- FIX: get all of the reviews
FIX: sometime the reviews have a lot of spaces
CHANGE: the method of get picture, now one less connection needed
CHANGE: one more value in Reviews option
CHANGE: several improvements in MyAnimeListUtils
[UPD][EN] MyAnimeList v1.3
[UPD][EN] MyAnimeList v1.3
MyAnimeList v1.3:
---------------------------------------MyAnimeListUtils.pas---------------------------------------
Code: Select all
unit MyAnimeListUtils;
uses StringUtils1;
const
HTML_CODES = '…=…,' + '«=«,' + '»=»,' + '‹=‹,' +
'›=›,' + '‘=‘,' + '’=’,' + '“=“,' +
'”=”,' + '‚=,,' + '„=„,' + ''='',' +
'"="';
ASCENDING = TRUE;
DESCENDING = FALSE;
//------------------------------------------------
function GetMovieName(Separator:string) : string;
begin
result := GetField(fieldOriginalTitle);
if (result = '') then begin
result := GetField(fieldTranslatedTitle);
if (Separator <> '') and (Pos(Separator,result) > 0 ) then
result := TextBefore(result,Separator,'');
end;
end;
//------------------------------------------------
function ValidatePercent(Value:string) : Boolean;
var
i, num: Integer;
ch: string;
begin
result := False;
for i := 1 to Length(Value) do begin
ch := Copy(Value,i,1);
if (Pos(ch,'0123456789') = 0) or (i > 3)
then EXIT;
end;
num := StrToInt(Value,0);
if (num >= 0) and (num <= 100)
then result := TRUE;
end;
//------------------------------------------------
procedure Sort(List:TStringList ; Invert:Boolean ; StartAt,EndAt:Integer);
var
str1, str2: string;
order: Boolean;
begin
if (EndAt <= StartAt) then EXIT;
str1 := List.GetString(StartAt);
str2 := List.GetString(StartAt + 1);
order := Length(str1) < Length(str2);
if Invert then order := not order;
if order and (Length(str1) <> Length(str2)) then begin
List.SetString(StartAt + 1,str1);
List.SetString(StartAt,str2);
end;
str1 := '';
str2 := '';
Sort(List,Invert,StartAt + 1,EndAt);
end;
//------------------------------------------------
procedure SortList(List:TStringList ; SortType:Boolean);
var
i: Integer;
begin
for i := 1 to List.Count do Sort(List,SortType,0,List.Count - i);
end;
//------------------------------------------------
function CompareText(Str1, Str2: string) :integer;
var
list1, list2: TStringList;
i, lineNr, charsMatch, totalChars:Integer;
begin
if (Str1 = '') or (Str2 = '') then begin
result := 0;
EXIT;
end;
list1 := TStringList.Create;
list2 := TStringList.Create;
list1.Text := StringReplace(Str1,' ',#13#10);
list2.Text := StringReplace(Str2,' ',#13#10);
SortList(list1,DESCENDING);
SortList(list2,DESCENDING);
for i := 0 to (list1.Count - 1) do begin
lineNr := FindLine(list1.GetString(i),list2,0);
if (lineNr > -1) then begin
charsMatch := charsMatch + Length(list1.GetString(i));
Str1 := StringReplace(Str1,list1.GetString(i),'');
list2.SetString(lineNr,#2);
end;
if (StringReplace(list2.Text,#2#13#10,'') = '') then Break;
end;
list1.Free;
list2.Free;
totalChars := Length(StringReplace(Str2,' ','')) + Length(StringReplace(Str1,' ',''));
result := (charsMatch * 100) div totalChars;
end;
//------------------------------------------------
function BlankAndCompact(Str:string ; CharsToBlank:string) : string;
var
ch: string;
atPos: Integer;
begin
for atPos := 1 to Length(CharsToBlank) do begin
ch := Copy(CharsToBlank,atPos,1);
Str := StringReplace(Str,ch,' ');
end;
while (Pos(' ',Str) > 0) do Str := StringReplace(Str,' ',' ');
result := Trim(Str);
end;
//------------------------------------------------
function RemoveNonAnsiHtmlNum(Str:string) : string;
var
text, htmlNum, htmlCode: string;
begin
htmlNum := TextBetween(Str,'&#',';');
if (htmlNum <> '') then
text := Copy(Str,1,Length(Str));
while (htmlNum <> '') do begin
htmlCode := '&#' + htmlNum + ';';
if (StrToInt(htmlNum,0) > 255) then
Str := StringReplace(Str,htmlCode,'?');
text := StringReplace(text,htmlCode,'');
htmlNum := TextBetween(text,'&#',';');
end;
result := Str;
end;
//------------------------------------------------
function HTMLDecodePatch(Str:string ; CodeTable:string) : string;
var
table: TStringList;
code, decode: string;
i: Integer;
begin
table := TStringList.Create;
table.CommaText := CodeTable;
for i := 0 to (table.Count - 1) do begin
code := table.GetName(i);
decode := TextAfter(table.GetString(i),'=');
Str := StringReplace(Str,code,decode);
end;
table.Free;
result := Str;
end;
//------------------------------------------------
function funcHTMLDecode(Str:string) : string;
begin
Str := HTMLDecodePatch(Str,HTML_CODES);
Str := RemoveNonAnsiHtmlNum(Str);
HTMLDecode(Str);
result := Str;
end;
//------------------------------------------------
function funcUTF8Decode(Str:string) : string;
begin
result := UTF8Decode(Str);
if (result = '') then result := Str;
end;
//------------------------------------------------
function funcHTMLRemoveTags(Str:string) : string;
begin
HTMLRemoveTags(Str);
result := Str;
end;
//------------------------------------------------
function TextDecode(Text:string ; StartAt:string ; EndAt:string) : string;
begin
result :=
FullTrim(
funcHTMLDecode(
funcUTF8Decode(
funcHTMLRemoveTags(
TextBetween(Text,StartAt,EndAt)
))));
end;
end.
---------------------------------------MyAnimeList.ifs---------------------------------------
Code: Select all
(***************************************************
Ant Movie Catalog importation script
www.antp.be/software/moviecatalog/
[Infos]
Authors=FinderX
Title=MyAnimeList
Description=Script for anime information in MyAnimeList.net
Site=http://myanimelist.net/
Language=EN
Version=1.3 @ 2010-06-13
Requires=3.5.1
Comments=In EDITOR MODE there're more options||DEFAULT_PERCENT USER_CHAR_SEPARATOR USER_SEPARATOR_LENGTH|ALT_TITLE_SEPARATOR INFO_START_TAG INFO_END_TAG
License=This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. |
GetInfo=1
[Options]
Batch Mode=0|0|0=No : Normal Mode|1=Yes : Batch Mode
Set % Once=0|0|0=No : Percent Default Value |1=Yes : Set Percent for one batch session
At Least One=0|0|0=No : No result if none comes to Percent|1=Yes : If none comes to Percent, then the best rate
Reviews=0|0|0=First two|1=First ten|2=All of them
Picture=0|0|0=Small : Thumbnail from Details Page|1=Large : Fullsize picture, if exists or else thumbnail
***************************************************)
program MyAnimeList;
uses MyAnimeListUtils;
const
DEFAULT_PERCENT = 100; // BatchMode Percentage Title, 100 = title perfect match
USER_CHAR_SEPARATOR = '_'; // Char separator between users in Comments
USER_SEPARATOR_LENGTH = 120; // Separator length between users in Comments
ALT_TITLE_SEPARATOR = ', '; // Here set your alternative titles separator
INFO_START_TAG = '['; // Additional Info start tag title section
INFO_END_TAG = ']'; // Additional Info end tag title section
BLANK_FOR_QUERY = ':;-·†_–%—''|`´~.,'; // More accurate results without these characters
MY_ANIME_LIST_QUERY = 'http://myanimelist.net/anime.php?q=';
MY_ANIME_LIST_ADDRESS = 'http://myanimelist.net/anime/';
FIND_MORE = 'Find More';
var
NumResults, BM_Percent, GetReviews:Integer;
BM_BestResultPercent: Integer;
MovieName, BM_BestResultAddress: string;
BM_MatchFind, BM_SetedPercentOnce: Boolean;
ExitNormalMode, SetPercent: Boolean;
BatchMode, LargePicture: Boolean;
CancelBatch, AtLeastOne: Boolean;
//------------------------------------------------
procedure SettingPercentOnce;
var
acceptValue: Boolean;
inputPercent:string;
begin
if BM_SetedPercentOnce then EXIT;
BM_SetedPercentOnce := TRUE;
BM_Percent := DEFAULT_PERCENT;
inputPercent := IntToStr(BM_Percent);
acceptValue := FALSE;
repeat
if (Input('Setting Percent','Percent [0 to 100]:',inputPercent) = FALSE) then begin
CancelBatch := TRUE;
EXIT;
end;
acceptValue := ValidatePercent(inputPercent);
if not acceptValue then
ShowError('The value "' + inputPercent + '" is not valid');
until acceptValue;
end;
//------------------------------------------------
procedure ExecutePickTree(var Address:string);
begin
if BatchMode then EXIT;
if (NumResults > 0) then begin
if PickTreeExec(Address) then begin
if (Address <> FIND_MORE) then begin
AnalyzeMoviePage(Address);
ExitNormalMode := TRUE;
end;
end else begin
Address := ''; // Cancel Button
end;
end else
ShowInformation('No results found for "' + MovieName +
'"' + #13#10 + 'Try another title');
end;
//------------------------------------------------
procedure AnalyzeForVariuosPages(Name:string);
var
address, nextAddress, addressPage: string;
strPage, titleTag: string;
pagePos, page, totalPages: integer;
begin
titleTag := '<title>Anime - MyAnimeList.net</title>';
address := MY_ANIME_LIST_QUERY + UrlEncode(BlankAndCompact(Name,BLANK_FOR_QUERY));
strPage := GetPage(address);
address := FIND_MORE;
pagePos := Pos('Pages (',strPage);
PickTreeClear;
if (pagePos > 0 ) then begin
addressPage := TextBetween(strPage,'Pages','</div>');
totalPages := StrToInt(TextBetween(addressPage,'(',')'),0);
for page := 1 to totalPages do begin
PickTreeAdd('MyAnimeList - Page ' + IntToStr(page) + ' of ' + IntToStr(totalPages),'');
AnalyzePage(strPage);
if (page < totalPages) then PickTreeMoreLink(FIND_MORE);
ExecutePickTree(address);
PickTreeClear;
if BM_MatchFind or (address <> FIND_MORE)
then Break;
if (page < totalPages) then begin
nextAddress := TextBetween(addressPage,'[' + IntToStr(page) + '] <a href="?q=','>');
nextAddress := MY_ANIME_LIST_QUERY + nextAddress;
strPage := GetPage(UrlEncode(nextAddress));
addressPage := TextBetween(strPage,'Pages','</div>');
end;
end;
end else begin
if (Pos(titleTag,strPage) > 0) then begin
PickTreeAdd('MyAnimeList','');
AnalyzePage(strPage);
ExecutePickTree(address);
end else begin
AnalyzeMoviePage(strPage);
ExitNormalMode := TRUE;
end;
end;
if BatchMode and AtLeastOne then
if (BM_MatchFind = FALSE) and (BM_BestResultAddress <> '') then
AnalyzeMoviePage(BM_BestResultAddress);
PickTreeClear;
end;
//------------------------------------------------
procedure AnalyzePage(PageStr:string);
var
movieAddr, movieTitle, movieID: string;
titleTag, title, compTitle, compName: string;
titleLine, compare: Integer;
strPage: TStringList;
begin
strPage := TStringList.Create;
strPage.Text := PageStr;
titleTag := '<a href="' + MY_ANIME_LIST_ADDRESS;
titleLine := FindLine(titleTag,strPage,0);
while (titleLine > -1) do begin
title := strPage.GetString(titleLine);
if (Pos('<strong>',title) > 0) then begin
movieId := TextBetween(title,titleTag,'/');
movieAddr := MY_ANIME_LIST_ADDRESS + MovieId;
movieTitle := TextDecode(title,'<strong>','</strong>');
if BatchMode then begin
compTitle := BlankAndCompact(movieTitle,BLANK_FOR_QUERY);
compName := BlankAndCompact(MovieName,BLANK_FOR_QUERY);
compare := CompareText(AnsiLowerCase(compName),AnsiLowerCase(compTitle));
if (compare > BM_BestResultPercent) then begin
BM_BestResultPercent := compare;
BM_BestResultAddress := movieAddr;
end;
end;
PickTreeAdd(movieTitle,movieAddr);
NumResults := NumResults + 1;
end;
titleLine := FindLine(titleTag,strPage,titleLine + 1);
end;
if BatchMode then begin
if (BM_Percent <= BM_BestResultPercent) then begin
AnalyzeMoviePage(BM_BestResultAddress);
BM_MatchFind := TRUE;
end;
end;
strPage.Free;
end;
//------------------------------------------------
procedure AnalyzeMoviePage(Address:string);
var
page: TStringList;
lineNr, lineAux, revLine, numRev: integer;
fansub, info, group, user, review, reviews: string;
reviewsAddress, pictureAddress, altTitle: string;
pictureTag, indexTag, userTag, groupTag: string;
item, thumbPicture, separator: string;
begin
fansub := '<a href="http://myanimelist.net/fansub-groups.php?id=';
thumbPicture := 'http://cdn.myanimelist.net/images/anime/';
pictureTag := '<a href="' + MY_ANIME_LIST_ADDRESS;
indexTag := '<a href="' + MY_ANIME_LIST_ADDRESS; // the doble space is important
userTag := '<a href="http://myanimelist.net/profile/';
page := TStringList.Create;
// URL
item := Copy(Address,1,Length('http://'));
if (Pos('http://',item) = 1) then begin
page.Text := GetPage(Address);
SetField(fieldURL,Address);
end else begin
page.Text := Address;
lineNr := FindLine(MY_ANIME_LIST_ADDRESS,page,0);
item := MY_ANIME_LIST_ADDRESS + TextBetween(page.GetString(lineNr),MY_ANIME_LIST_ADDRESS,'/');
SetField(fieldURL,item);
end;
// Original Title - Title
lineNr := FindLine('<title>',page,0);
SetField(fieldOriginalTitle,TextDecode(page.GetString(lineNr),'<title>',' - MyAnimeList.net</title>'));
// Picture
lineNr := FindLine(thumbPicture,page,lineNr);
pictureAddress := thumbPicture + TextBetween(page.GetString(lineNr),thumbPicture,'" alt="');
if CanSetPicture then begin
if LargePicture and (Pos('pic&pid',page.GetString(lineNr)) > 0) then
pictureAddress := StringReplace(pictureAddress,'.jpg','l.jpg');
GetPicture(pictureAddress);
end; // at this point lineNr is used consecutive for better performance
// TranslatedTitle - Alternative Title(s)
lineNr := FindLine('Alternative Titles',page,lineNr);
altTitle := TextDecode(page.GetString(lineNr),'English:','</div>');
item := TextDecode(page.GetString(lineNr),'Synonyms:','</div>');
if (item <> '') then begin
item := StringReplace(item,', ',ALT_TITLE_SEPARATOR);
if (altTitle <> '') then altTitle := altTitle + ALT_TITLE_SEPARATOR;
altTitle := altTitle + item;
end;
item := TextDecode(page.GetString(lineNr),'Japanese:','</div>');
if (item <> '') then begin
item := StringReplace(item,', ',ALT_TITLE_SEPARATOR);
if (altTitle <> '') then altTitle := altTitle + ALT_TITLE_SEPARATOR;
altTitle := altTitle + item;
end;
SetField(fieldTranslatedTitle,altTitle);
// Director - Type
lineNr := FindLine('Type:',page,lineNr);
SetField(fieldDirector,TextDecode(page.GetString(lineNr),'Type:','</div>'));
// Disks - Episodies
lineNr := FindLine('Episodes:',page,lineNr);
item := page.GetString(lineNr) + #2;
SetField(fieldDisks,TextDecode(item,'Episodes:',#2));
// Year - Year of start aired only
lineNr := FindLine('Aired:',page,lineNr);
item := Copy(TextAfter(page.GetString(lineNr),', '),1,4);
SetField(fieldYear,item);
// Productor - Studio
lineNr := FindLine('Producers:',page,lineNr);
item := page.GetString(lineNr) + page.GetString(lineNr + 1);
if (Pos('None found',item) = 0)
then item := TextDecode(item,'Producers:','</div>')
else item := '';
SetField(fieldProducer,item);
// Source - Genres
lineNr := FindLine('Genres:',page,lineNr);
item := page.GetString(lineNr) + page.GetString(lineNr + 1);
item := AnsiMixedCase(TextDecode(item,'Genres:','</div>'),' ,-');
SetField(fieldSource,StringReplace(item,' Of ',' of '));
// Reviews Address
lineNr := FindLine(indexTag,page,lineNr);
while (Pos('Reviews',page.GetString(LineNr)) = 0) do
lineNr := FindLine(indexTag,page,lineNr + 1);
reviewsAddress := MY_ANIME_LIST_ADDRESS + TextBetween(page.GetString(lineNr),indexTag,'">');
// Description - Synopsis
lineNr := FindLine('<h2>Synopsis</h2>',page,lineNr);
item := page.GetString(lineNr);
while (Pos('</td>',item) = 0) do begin
lineNr := lineNr + 1;
item := item + page.GetString(lineNr);
end;
item := StringReplace(item,'<br />',#13#10);
SetField(fieldDescription,TextDecode(item,'<h2>Synopsis</h2>','</td>'));
// Actors - Additional information - Related Anime
lineAux := FindLine('Related Anime',page,lineNr);
if (lineAux > -1) then begin
info := INFO_START_TAG + 'Related Anime' + INFO_END_TAG + #13#10;
item := page.GetString(lineAux) + #2;
item := StringReplace(item,'<br>',#13#10);
info := info + TextDecode(item,'Related Anime',#2) + #13#10 + #13#10;
lineNr := lineAux;
end;
// Number of Reviews, in reviews page
lineNr := FindLine('More reviews (',page,lineNr);
numRev := StrToInt(TextBetween(page.GetString(lineNr),'More reviews (',')'),0);
lineAux := lineNr; // lineAux is used later in Comments
// Actors - Additional information - Opening Theme
lineNr := FindLine('Opening Theme',page,lineNr);
item := page.GetString(lineNr) + #2;
if (Pos('No opening themes found',item) = 0) then begin
info := info + INFO_START_TAG + 'Opening Theme' + INFO_END_TAG + #13#10;
item := TextDecode(item,'Opening Theme',#2);
item := StringReplace(item,'#',#13#10 + '#');
info := info + FullTrim(item) + #13#10 + #13#10;
end;
// Actors - Additional information - Ending Theme
lineNr := FindLine('Ending Theme',page,lineNr);
item := page.GetString(lineNr) + #2;
if (Pos('No ending themes found',item) = 0) then begin
info := info + INFO_START_TAG + 'Ending Theme' + INFO_END_TAG + #13#10;
item := TextDecode(item,'Ending Theme',#2);
item := StringReplace(item,'#',#13#10 + '#');
info := info + FullTrim(item) + #13#10 + #13#10;
end;
// Actors - Additional information - Fansubs
lineNr := FindLine(fansub,page,lineNr); // last use for consecutive lineNr
if (lineNr > -1) then
info := info + INFO_START_TAG + 'Fansubbing Groups' + INFO_END_TAG + #13#10;
while (lineNr > -1) do begin
group := TextBefore(page.GetString(lineNr),'</a>','') + #2;
info := info + BlankAndCompact(TextDecode(group,'>',#2),#9#13#10);
groupTag := TextDecode(page.GetString(lineNr),'<small>[',']</small>');
if (groupTag <> '') then
info := info + ' - [' + BlankAndCompact(groupTag,#9#13#10) + ']';
groupTag := TextDecode(page.GetString(lineNr),'<small>(',')</small>');
if (groupTag <> '') then
info := info + ' - (' + BlankAndCompact(groupTag,#9#13#10) + ')';
info := info + #13#10;
lineNr := FindLine(fansub,page,lineNr + 1);
end;
SetField(fieldActors,FullTrim(info));
// User Separator for Comments
if (USER_CHAR_SEPARATOR <> '') then begin
separator := StringOfChar(USER_CHAR_SEPARATOR,USER_SEPARATOR_LENGTH);
if (separator = '') or (USER_CHAR_SEPARATOR = '_')
then separator := separator + #13#10
else separator := #13#10 + separator + #13#10;
end else
separator := #13#10;
// Comments - Reviews
if CanSetField(fieldComments) then begin
repeat
if (GetReviews > 0) and (numRev > 0) then begin
// Destroy Details or Reviews Page
page.Free;
// Create New Reviews Page
page := TStringList.Create;
page.Text := GetPage(reviewsAddress);
lineNr := FindLine('More Reviews</a>',page,0);
if (lineNr > -1) then
reviewsAddress := TextBefore(reviewsAddress,'reviews','') +
TextBefore(page.GetString(lineNr),'">More Reviews</a>','<a href="');
lineAux := lineNr;
end else begin
lineNr := lineAux; // use lineAux before mentioned
lineAux := -1; // exit loop
end;
lineNr := FindLine(userTag,page,lineNr);
while (lineNr > -1) do begin
user := page.GetString(lineNr);
if (Pos('<br />',user) > 0) then begin
user := TextDecode(user,'">','</a>');
reviews := reviews + user + ':' + #13#10 + #13#10;
revLine := FindLine('</table>',page,lineNr);
repeat
if (Pos(#9#9 + '</div>',page.GetString(revLine + 1)) = 1) then begin
repeat
review := review + page.GetString(revLine);
revLine := revLine + 1;
until (Pos('<div>',review) > 0);
review := StringReplace(review,'<br />',#13#10);
review := StringReplace(review,'read more</a>',' ');
review := TextDecode(review,'</table>','<div>');
review := BlankAndCompact(review,''); // compact spaces only
reviews := reviews + review + #13#10;
reviews := reviews + separator + #13#10;
review := '';
Break;
end;
revLine := FindLine('</table>',page,revLine + 1);
until (revLine = -1);
end;
lineNr := FindLine(userTag,page,lineNr + 1);
end;
until (lineAux = -1) or (GetReviews < 2);
SetField(fieldcomments,FullTrim(reviews));
end;
// Destroy Details or Reviews Page
page.Free;
end;
//----------------------MAIN-PROGRAM-------------------------
begin
if (CheckVersion(3,5,1)=FALSE) then begin
ShowWarning('Required Ant Movie Catalog version 3.5.1 or higher');
EXIT;
end;
if CancelBatch then EXIT; // For remaining batch
GetReviews := GetOption('Reviews');
LargePicture := GetOption('Picture') = 1;
BatchMode := GetOption('Batch Mode') = 1;
SetPercent := GetOption('Set % Once') = 1;
AtLeastOne := GetOption('At Least One') = 1;
BM_MatchFind := FALSE;
BM_BestResultPercent := -1; // 0 is also an option
BM_BestResultAddress := '';
ExitNormalMode := FALSE;
NumResults := 0;
MovieName := GetMovieName(ALT_TITLE_SEPARATOR);
if BatchMode then begin
if SetPercent
then SettingPercentOnce
else BM_Percent := DEFAULT_PERCENT;
if CancelBatch then EXIT; // For Cancel in SettingPercentOnce window
AnalyzeForVariuosPages(MovieName);
end else begin // NormalMode
repeat
if Input('MyAnimeList', 'Search:', MovieName)
then AnalyzeForVariuosPages(MovieName)
else ExitNormalMode := TRUE;
until ExitNormalMode;
end;
end.
Last edited by FinderX on 2010-06-13 20:33:35, edited 1 time in total.