Code: Select all
(***************************************************
Ant Movie Catalog importation script
www.antp.be/software/moviecatalog/
[Infos]
Authors=Fulvio53s03
Title=wikipedia.it
Description=import film e serie TV
Site=http://www.wikipedia.it
Language=IT
Version=0.2
Requires=4.2.1
Comments=puo' essere usato per ricerca informazioni tramite google (mode=0) oppure per estrarle direttamente da wikipedia.it (mode=1) inserendo l'URL direttamente. |Inserire il termine (Stagione....) nel titolo tradotto rende migliore la ricerca della cover in bing.com per le serie TV
License=* The source code of the script can be used in |* another program only if full credits to Fulvio53s03*
GetInfo=1
RequiresMovies=1
[Options]
Mode=0|0|0=normal mode|1=batch mode (url)
Origine_foto=1|0|0=img da wiki|1=img da bing
[Parameters]
***************************************************)
// TypeSearch=1|1|1=search for exact expression|2=search for any word
// TypeOrder=3|3|1=order by Artist|2=order by film_o_serie Title|3=order by relevance
// needs the following units
// StringUtils7552.pas
//
program wiki;
uses
StringUtils7552;
const
UrlBase = 'https://it.wikipedia.org'; // base url
bing_Base = 'https://www.bing.com/images/search?q=';
debug_search = false; // debug mode on/off su ricerca files
debug_film_o_serie = false; // debug mode on/off su estrazione dati film_o_serie
folder = 'f:\prova\'; // directory where to save files
CRLFspace = CRLF + ' ';
//2016-02-10 tappo_fine_ricerca = '<li class="g">Tappodifinericercadaticongoogle</li><li class="g">';
tappo_fine_ricerca = '<div class="g">Tappodifinericercadaticongoogle</div><div class="g">'; //2016-02-10
MaxItems = 50;
var
nuovo_anno, mese_anno, solo_anno, data_rilascio, label, stagione: String;
Credit_address, Readdress, Page: String;
film_o_serieName, ArtistName, firstcall, name, autori: String;
Formato, Releases, artista: String;
file_name, campo_URL: String;
tipo_ricerca, pag_ricerca, save_value, ordine_lista, Salva_autore: String;
value, episodi, lista_episodi, riquadro: string;
CharNormal, CharAbNormal: string;
film_o_serie_ok: boolean;
BatchMode, origine_foto_opt, giri, anno_int: Integer;
pos_chr, lgth_file_name: Integer;
titolo_e_autore, titolo, autore, durata, cover: String;
save_comments, commento_episodi, commento_skeda: String;
batchlog, confbatch: TstringList;
save_pag, artist, initchar, endchar: string;
film_o_serieok: boolean;
bing_url, stagione_special: string;
//------------------------------------------------------------------------------
// list of film_o_series
//------------------------------------------------------------------------------
procedure GetList;
var
Address, urlmusic, album, albumtest, genre: String;
Save_address, Title_album, indir_album, indir_title_album: String;
dati_album, init_control, end_control, init_ol, end_ol: String;
Desc_ricerca: String;
ctr_loop: integer;
found: boolean;
i, lgth_dati_album: integer;
begin
PickTreeClear; // clear list
desc_ricerca := file_name;
if desc_ricerca = '' then
desc_ricerca := film_o_serieName;
//fs2017.01.30 Address := 'https://www.google.it/search?as_q=' + film_o_serieName;
Address := 'https://www.google.it/search?num=20&as_q=' + film_o_serieName; //fs2017.01.30
Address := Address + '&as_epq=&as_oq=&as_eq=&as_nlo=&as_nhi=&lr=&cr=&as_qdr=all';
Address := Address + '&as_sitesearch=it.wikipedia.org&as_occt=any&safe=images&as_filetype=&as_rights=';
Address := UrlEncode(Address);
Page := GetPage(Address);
Save_address := Address;
SetField(fieldURL, Address);
HTMLdecode(Page);
if debug_search then
DumpPage(folder+'googleListPage.html', Page); // debug
found := True;
// PickTreeAdd('List of albums found for' + '"' + AlbumName + '"', '');
Desc_ricerca := 'Lista articoli trovati per -' + Desc_ricerca + '"';
PickTreeAdd(Desc_ricerca, '"');
if debug_search then
DumpPage(folder+'wikiList.txt', Page); // debug
if Pos('find anything', Page) <> 0 then
begin
LogMessage('Error while reading selection page - no results found for ' + Desc_ricerca);
Found := False;
exit;
end;
init_ol := '<ol>'; //2015-11-25
end_ol := '</ol>';
pag_ricerca := TextBetween(Page, init_ol, end_ol) + tappo_fine_ricerca + end_ol; //elenco titoli richiesti e tappo ricerca
if debug_search then
DumpPage(folder+'wikiricerca.html', pag_ricerca); // debug
save_pag := pag_ricerca;
init_control := '<div class="g">'; //2016-02-10
end_control := '<div class="g">'; //2016-02-10
dati_album := init_control + TextBetween (pag_ricerca, init_control, end_control) + end_control;
//dati_album contiene recursivamente i dati delle pagine referenziate da google
lgth_dati_album := length(dati_album);
if debug_search then
DumpPage(folder+'google_dati_album.txt', dati_album); // debug
delete(Pag_ricerca, 1, (lgth_dati_album - length(end_control)));
if debug_search then
DumpPage(folder+'wikiricerca-1.txt', Pag_ricerca); // debug
while (dati_album <> tappo_fine_ricerca) do //estraggo fino al tappo di fine ricerca
begin
//*** ricerca dell'indirizzo da linkare (N.B. prima devi cercare https, poi cercare http
initchar := 'https://'; endchar := '/%252';
Address := textBetween(dati_album, initchar, endchar);
if length(address) < 1 then
begin
initchar := 'https://'; endchar := '&sa';
Address := textBetween(dati_album, initchar, endchar);
end;
if length(address) < 1 then
begin
initchar := 'http://'; endchar := '/&252';
Address := textBetween(dati_album, initchar, endchar);
if length(address) < 1 then
begin
initchar := 'http://'; endchar := '&sa';
Address := textBetween(dati_album, initchar, endchar);
end;
end;
Address := initchar + Address;
//*** fine ricerca indirizzo
initchar := '<h3 class="r">';
endchar := '</h3>';
Title_album := textBetween(dati_album,initchar, endchar); //descrizione del link
HTMLRemoveTags(Title_album);
// Title_album := copy(Title_album, 1, 42);
PickTreeAdd(Title_Album, Address);
found := True;
init_control := '<div class="g">'; //2016-02-10
end_control := '<div class="g">'; //2016-02-10
dati_album := init_control + TextBetween (pag_ricerca, init_control, end_control) + end_control;
lgth_dati_album := length(dati_album);
delete(Pag_ricerca, 1, (lgth_dati_album - length(end_control)));
lgth_dati_album := length(Dati_album) - (length(init_control) + length(end_control));
end;
// end;
if not found then
begin
LogMessage('No album found for ' + Desc_ricerca);
exit;
end;
if PickTreeExec(Address) then
begin
AnalyzeMoviePage(Address); // Album page
end
else
LogMessage('No serie/film selected');
// ---------------------------------------
end;
//------------------------------------------------------------------------------
// ANALYZE Movie PAGE
//------------------------------------------------------------------------------
procedure AnalyzeMoviePage(Address: string);
var
urlmusic, Lenght, Pays: String;
Autore, style, Commenti, str_min, campo_note: String;
minu, minuti, ore, lgth_comm, save_lgth_comm, i: integer;
j: Real;
begin
save_comments := '';
commento_episodi := '';
commento_skeda := '';
estrazione_scheda(Address);
// save_comments := commento_skeda + CRLF + commento_episodi;
// SetField(fieldcomments, save_comments); // commenti + elenco episodi end
end;
//------------------------------------------------------------------------------
// ESTRAE INFORMAZIONI DA SCHEDA
//------------------------------------------------------------------------------
procedure estrazione_scheda(Address: string);
begin
Page := GetPage(Address);
SetField(fieldURL, Address);
Page := UTF8decode(Page);
HTMLdecode(Page);
if debug_film_o_serie then
DumpPage(folder+'wikiPageDetail.txt', Page); // debug_film_o_serie
if (BatchMode = 0) and (tipo_ricerca = 'sk ep') then
SetField(fieldURL, Address);
campo_URL := getfield(fieldurl);
SetField(fieldDate, DateToStr(Date));
Normalizza_Page(Page);
if debug_film_o_serie then
DumpPage(folder+'wikiPageDetailHTMLdecode.txt', Page); // debug_film_o_serie
// Page := UTF8Decode(Page); //fs2015-01-15
film_o_serieok := True;
Value := Page;
//fs 2017.01.28 riquadro := textbetween(Value, '<div id="mw-content-text"', '</div>');
riquadro := textbetween(Value, '<table class="sinottico"', '</table>'); //fs 2017.01.28
if debug_film_o_serie then
DumpPage(folder+'wikiriquadro.txt', riquadro);
//*** titolo tradotto
initchar := '<title>'; //fs2016-06-26
endchar := '</title>';
label := textbetween(Value, initchar, endchar);
label := stringreplace(label, ' - Wikipedia', '');
if getfield (fieldtranslatedTitle) = '' then
SetField(fieldtranslatedTitle, label);
//*** Descrizione (trama)
label := textbetween(value, '<span class="mw-headline" id="Trama">Trama</span>', '<h');
label := textbetween(label, '<p>', '</p>');
// label := textbetween(value, '<h2><span class="mw-headline" id="Trama">', '<h2>');
if label = '' then //2017.03.11 FS
begin
label := textbetween(value, '<span class="mw-headline" id="Descrizione">Descrizione</span>', '<h');
label := textafter(label, '</h2>');
end;
if debug_film_o_serie then
DumpPage(folder+'wikiDescrizione.html', label); // debug_film_o_serie
if label = '' then
begin
label := textafter(value, '</table>');
label := textbefore(label, '<h2>', '');
end;
HTMLRemoveTags(label);
label := fulltrim(label);
SetField(fielddescription, label);
//*** commento (descrizione generale)
initchar := '<p><i><b>';
label := initchar + textbetween(value, initchar, '<div id="toc" class="toc">');
// label := textbetween(label, '<p>', '</p>');
HTMLRemoveTags(label);
SetField(fieldcomments, label);
// *** dati estratti dal riquadro sinottico ***
//*** country
initchar := '>Paese'; endchar := '</tr>';
label := TextBetween(riquadro, initchar, endchar);
HTMLRemoveTags(label);
label := stringReplace(label, 'di produzione', '');
label := stringReplace(label, 'Stati Uniti d''America', 'USA');
label := fulltrim(label);
SetField(fieldcountry, label);
//*** cover locandina da wikipedia ------------------
cover := 'https:' + TextBetween(riquadro, 'src="', '"');
if origine_foto_opt = 0 then
begin
if cover <> 'https:' then
GetPicture(cover);
end;
//*** cover locandina da Bing -------------------
// bing_url := bing_Base + UrlEncode(film_o_serieName) + '&qft=+filterui:imagesize-large+filterui:aspect-tall&FORM=R5IR28';
// delete(Pag_ricerca, 1, (lgth_dati_album - length(end_control)));
pos_chr := 0;
pos_chr := pos('(stagione-)', film_o_serieName);
if pos_chr = 0 then
pos_chr := pos('(stagione-', film_o_serieName);
if pos_chr = 0 then
pos_chr := pos('(stagione', film_o_serieName);
if pos_chr = 0 then
pos_chr := pos(' stagione ', film_o_serieName);
if pos_chr = 0 then
pos_chr := pos(' serie ', film_o_serieName);
lgth_file_name := length(film_o_seriename);
if pos_chr = 0 then
begin
bing_url := bing_Base + UrlEncode('COVER ' + film_o_serieName);
bing_url := bing_url + '&qft=+filterui:imagesize-large+filterui:aspect-tall&FORM=R5IR28';
end;
if pos_chr > 0 then
begin
// bing_url := delete(film_o_seriename, pos_chr, (lgth_file_name - pos_chr));
bing_url := film_o_seriename;
delete(bing_url, (pos_chr - 1), (lgth_file_name - pos_chr + 2));
bing_url := bing_Base + UrlEncode('SERIE ' + bing_url);
bing_url := bing_url + '&qft=+filterui:imagesize-large+filterui:aspect-tall&FORM=R5IR28';
end;
if origine_foto_opt = 1 then
AnalyzeBing(bing_url);
//----------------- fine estrae cover da Bing -------------------
//*** Titolo Originale
label := textbetween(riquadro, '>Titolo originale<', '</td>');
initchar := '<td'; endchar := '</td>';
label := initchar + textbetween(label, initchar, '</i>');
label := FormatText(label);
SetField(fieldoriginalTitle, label);
//*** genere
initchar := '>Genere<';
label := textbetween(riquadro, initchar, '</tr>');
initchar := '<td'; endchar := '</td>';
label := initchar + textbetween(label, initchar, endchar);
HTMLRemoveTags(label);
label := AnsiMixedCase(AnsiLowerCase(label), ' ');
SetField(fieldcategory, label);
//*** regia
initchar := '>Regia</a>'; endchar := '</td>';
label := TextBetween(riquadro, initchar, endchar);
HTMLRemoveTags(label);
label := FormatText(label);
SetField(fieldDirector, label);
//*** personaggi e interpreti
initchar := '<ul>';
endchar := '</ul>';
label := initchar + TextBetween(riquadro, initchar, endchar) + endchar;
initchar := '<li>';
label := initchar + TextBetween(riquadro, initchar, endchar) + endchar;
HTMLRemoveTags(label);
SetField(fieldactors, label);
//*** anno
initchar := '>Anno'; endchar := '</a>';
label := TextBetween(riquadro, initchar, endchar);
label := stringReplace(label, ' - in produzione', '');
HTMLRemoveTags(label);
// label := FormatText(label);
SetField(fieldyear, label);
label := getfield(fieldyear);
//*** produttore
initchar := '>Casa di produzione<'; endchar:= '</tr>';
label := textbetween(riquadro, initchar, endchar);
label := '<' + stringReplace(label, '<br />', ', ');
HTMLRemoveTags(label); // label := stringReplace(label, ' - in produzione', '');
SetField(fieldproducer, label);
//*** writer (ideatore della serie TV) - soggetto (film)
initchar := '>Soggetto'; //film
label := textbetween(value, initchar, '</tr>');
// label := textbetween(label, '<td>', '</td>');
HTMLRemoveTags(label);
SetField(fieldwriter, label);
if label = '' then
begin
initchar := '<th>Ideatore</th>'; //serie
label := textbetween(value, initchar, '</tr>');
// label := textbetween(label, '<td>', '</td>');
HTMLRemoveTags(label);
SetField(fieldwriter, label);
end;
//*** compositore
initchar := '>Musiche'; endchar:= '</tr>';
label := textbetween(riquadro, initchar, endchar);
label := '<' + stringReplace(label, '<br />', ', ');
HTMLRemoveTags(label); // label := stringReplace(label, ' - in produzione', '');
SetField(fieldcomposer, label);
end;
Procedure Normalizza_page(Pagina: string); // elimina i crlf, trasforma delimiters maiuscoli in minuscoli
begin
pagina := RegExprSetReplace('<([^>]+)>', pagina, '<\L\1>', true);
CharAbNormal := crlf; CharNormal := ' ';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '<B'; CharNormal := '<b';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</B'; CharNormal := '</b';
pagina := StringReplace(Value, CharAbNormal, CharNormal);
CharAbNormal := '<FONT'; CharNormal := '<font';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</FONT'; CharNormal := '</font';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '<TR'; CharNormal := '<tr';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</TR'; CharNormal := '</tr';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '<TD'; CharNormal := '<td';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</TD'; CharNormal := '</td';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '<DIV'; CharNormal := '<div';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</DIV'; CharNormal := '</div';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '<Ol'; CharNormal := '<ol';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</Ol'; CharNormal := '</ol';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
end;
//------------------------------------------------------------------------------
// set show warning (normal mode) or add to log (batch mode)
//------------------------------------------------------------------------------
procedure LogMessage(m: string);
begin
if BatchMode > 0 then
AddToLog('item '+GetField(fieldNumber)+': '+m)
else
ShowWarning(m);
end;
//------------------------------------------------------------------------------
// add a message in the batch log and save to disk
// (because I don't know when it's finished...)
//------------------------------------------------------------------------------
procedure AddToLog(m: string);
begin
//fs2016-12-31 batchlog.Add(m);
//fs2016-12-31 batchlog.SaveToFile(batchlogfic);
end;
//------------------------------------------------------------------------------
// process batch mode
//------------------------------------------------------------------------------
procedure wikiBatch;
begin
film_o_serieName := GetField(fieldUrl); // if no url or another site then ignore
if (film_o_serieName <> '') and (Pos(UrlBase, film_o_serieName) > 0) then
AnalyzeMoviePage(film_o_serieName)
else
LogMessage('ignored url="'+film_o_serieName+'"');
end;
//------------------------------------------------------------------------------
// process normal mode
//------------------------------------------------------------------------------
procedure wikiNorm;
begin
file_name:= getField(fieldfilepath);
if (GetField(fieldTranslatedTitle) <> '') then
film_o_serieName := GetField(fieldTranslatedTitle) // get film_o_serie name
else
film_o_serieName := GetField(fieldOriginalTitle); //mrobama
repeat
if not Input('wiki = ' + ArtistName + ' - ' + film_o_serieName, 'Scrivi il nome del film o della serie:'
+ crlf, film_o_serieName) or (film_o_serieName = '') then exit;
formato := getfield(FieldOriginalTitle);
if pos(formato, 'serie') > 0 then
formato:= 'serie';
GetList;
until film_o_serieok;
end;
//---------------------------- estrazione immagine da Bing ------------------------
// ---
function RemovePar(wholetext: string) : string;
var
str1: String;
i: Integer;
begin
str1 := Trim(TextBefore(wholetext, '(', ''));
if str1 <> '' then
begin
if Pos(')', RemainingText) > 0 then
wholetext := str1+' '+Trim(TextAfter(RemainingText, ')')); // + end of text or ''
end;
result := Trim(wholetext);
end;
// ---
procedure AnalyzeBing(bing_Address: string);
var
ImgCnt: Integer;
PageText,strRef,strImg,strTest, Title_photo: string;
begin
// Read the whole page
PageText := GetPage(bing_Address);
if debug_search then
DumpPage(folder+'bingSearchPage.html', PageText); // debug
initchar := '<a class="thumb"'; endchar := '<div class="fileInfo">';
strTest := initchar + TextBetween(PageText, initchar, endchar) + endchar;
if (strTest = '') then
break;
// Check if there is a class with the url inside this href
if pos('class="thumb"', strTest) > 0 then //class="thumb"
begin
// Get the img reference
initchar := 'href="'; endchar := '"';
strImg := TextBetween(strTest, initchar, endchar);
initchar := '<div class="des">'; endchar := '</div>';
Title_photo := Textbetween(strtest, initchar, endchar);
Title_photo:= FormatText(Title_photo);
GetPicture (strImg);
end;
end;
//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
// start here
//------------------------------------------------------------------------------
begin
if not CheckVersion(4,2,1) then
begin
ShowMessage('This script requires a newer version of Ant Movie Catalog (at least the version 4.2.1)');
exit;
end;
// get user's parms (used more than once)
BatchMode := GetOption('Mode');
origine_foto_opt := GetOption('Origine_foto');
// *********************************************************************
if BatchMode = 1 then
wikiBatch;
if BatchMode = 0 then
wikiNorm;
end.
