Code: Select all
(***************************************************
Ant Movie Catalog importation script
www.antp.be/software/moviecatalog/
[Infos]
Authors=Fulvio53s03
Title=Discogs
Description=import album
Site=http://www.discogs.com
Language=IT, EN
Version=1.4.2
Requires=4.2.0
Comments=It can be used to search informations (mode=0) and to extract them to your AMC d.b. or to extract directly informations (mode=1), if you already know your album address' page in discogs. For a better selecting informations, you can also supply Artist's name in Original Title field and the information: CD, Album, LP ecc. in Media Type field.
License=* The source code of the script can be used in |* another program only if full credits to Fulvio53s03 |* script
GetInfo=1
RequiresMovies=1
[Options]
Mode=0|0|0=normal mode|1=batch mode (url)
[Parameters]
***************************************************)
// TypeSearch=1|1|1=search for exact expression|2=search for any word
// TypeOrder=3|3|1=order by Artist|2=order by Album Title|3=order by relevance
// needs the following units
// StringUtils7552.pas
//
program discogs;
uses
StringUtils7552;
const
discogsUrl = 'http://www.discogs.com'; // base url
//<div class="pagination_controls "> inizio links
//<div class="pagination bottom"> fine links
//<div class="cards cards_layout_large" id="search_results"> inizio struttura interna link
AllAlbumUrl = 'http://www.discogs.com'; // base url
batchlogfic = 'c:\discogs_batch.txt'; // log for batch mode
debug_search = false; // debug mode on/off su ricerca dischi
debug_album = false; // debug mode on/off su estrazione dati album
folder = 'c:\'; // directory where to save files
inizio_e_fine_traccia = '<td class="tracklist_track_pos">';
inizio_credits = '<li>';
fine_credits = '</li>';
CRLFspace = CRLF + ' ';
var
nuovo_anno, mese_anno, solo_anno, data_rilascio, label: String;
Credit_address, Readdress, Page: String;
AlbumName, ArtistName, firstcall, abort, name, Dati_traccia, autori: String;
Formato, Releases, artista: String;
pag_ricerca, Commento, save_value, ordine_lista, Salva_autore: String;
value: string;
CharNormal, CharAbNormal: string;
Album_ok: boolean;
BatchMode, index, lungo: Integer;
brani, Track, newtrack, old_track, datitrack: String;
titolo_e_autore, titolo, autore: String;
batchlog, confbatch: TstringList;
save_pag, artist, initchar, endchar: string;
albumok: boolean;
//------------------------------------------------------------------------------
// list of albums
//------------------------------------------------------------------------------
procedure GetList;
var
Address, urlmusic, album, albumtest, label, genre: String;
Save_address, Title_album, indir_album, indir_title_album: String;
Relevance, info_disco, artist_album, tipo_release, altri_dati: String;
dati_album, init_control, end_control: String;
Desc_ricerca: String;
ctr_loop: integer;
found: boolean;
i, lgth_dati_album, lgth_pag_ricerca: integer;
// http://www.discogs.com/search/?q=bitches+brew
begin
PickTreeClear; // clear list
// http://www.discogs.com/search/?q=Symphony+X&type=all
//fs20150727 Address := discogsUrl + '/search/?q=' + UrlEncode(AlbumName) + '&layout=sm'; //fs
Address := discogsUrl + '/search/?q=' + artista + ' ' + AlbumName + '&layout=sm'; //fs20150727
//fs2015.06.10 Address := discogsUrl + '/search/?q=' + AlbumName + ' ';
//fs2015.06.10 Address := Address + ArtistName + '&format_exact=' + formato + '&layout=sm'; //fs
// Address := discogsUrl + '/search/?q=' + AlbumName + '&type=all'; //fs2015.06.10
Address := UrlEncode(Address);
Desc_ricerca := '"' + AlbumName + '"';
if length(ArtistName) > 0 then
Desc_ricerca := Desc_ricerca + ', ' + ArtistName;
if length(formato) > 0 then
Desc_ricerca := Desc_ricerca + ', ' + formato;
Page := GetPage(Address);
Save_address := Address;
SetField(fieldURL, Address);
HTMLdecode(Page);
found := False;
// PickTreeAdd('List of albums found for' + '"' + AlbumName + '"', '');
PickTreeAdd('List of albums found for ' + Desc_ricerca, '');
if debug_search then
DumpPage(folder+'discogsList.txt', Page); // debug
if Pos('find anything', Page) <> 0 then
begin
LogMessage('Error while reading selection page - no results found for ' + Desc_ricerca);
exit;
end;
init_control := '<div class="pagination_controls ">';
end_control := '<div class="pagination bottom ">';
info_disco := '<div class="card_body">';
pag_ricerca := init_control + TextBetween(Page, init_control, end_control) + end_control; //elenco titoli richiesti
lgth_pag_ricerca := length(pag_ricerca);
Normalizza_Page(pag_ricerca);
save_pag := pag_ricerca;
if debug_search then
DumpPage(folder+'discogs_pag_ricerca.txt', pag_ricerca); // debug
dati_album := info_disco + TextBetween (pag_ricerca, info_disco, info_disco) + info_disco;
//dati_album contiene recursivamente i dati degli album
lgth_dati_album := length(dati_album);
if debug_search then
DumpPage(folder+'discogs_dati_album.txt', dati_album); // debug
delete(Pag_ricerca, 1, length(dati_album) - length(endchar));
ctr_loop := 0;
lungo := (Length(info_disco) + length(info_disco));
while (lgth_dati_album > Lungo) and (ctr_loop < 50) do //estraggo al massimo 50 titoli
begin
ctr_loop := ctr_loop + 1;
initchar := '<a class="search_result_title';
endchar := '</a>';
indir_title_album := textBetween(dati_album, initchar, endchar) + endchar;
initchar := 'href="';
endchar := '"';
Address := discogsUrl + textBetween(indir_title_album,initchar, endchar);
initchar := '<a href="/artist';
endchar := '</span>';
artist_album := textBetween(dati_album,initchar, endchar); //Artista
initchar := '>';
endchar := '<';
artist_album := textBetween(artist_album,initchar, endchar); //Artista
initchar := '>';
endchar := '</a>';
Title_album := textBetween(indir_title_album,initchar, endchar); //titolo dell'album
initchar := '<p class="card_status">';
endchar := '</p>';
tipo_release := textbetween(dati_album, initchar, endchar);
tipo_release := formatText(tipo_release); //tipo release
initchar := '<p class="card_info">';
endchar := '</p>';
altri_dati := textbetween(Dati_album, initchar, endchar);
altri_dati := formatText(altri_dati);
Title_Album := artist_album + ' - ' + Title_Album + ' (' + tipo_release + ', ' + altri_dati + ')';
Title_Album := stringreplace(Title_Album, CRLF, '');
Title_Album := stringreplace(Title_Album, ' Needs Vote,', '');
PickTreeAdd(Title_Album, Address);
found := True;
delete(Pag_ricerca, 1, lgth_dati_album);
Pag_ricerca := Pag_ricerca + info_disco; //fs 2015_01_12
lgth_pag_ricerca := length(Pag_ricerca);
dati_album := info_disco + TextBetween (pag_ricerca, info_disco, info_disco); //recursivamente i dati degli album
lgth_dati_album := length(Dati_album);
end;
if not found then
begin
LogMessage('No album found for '+AlbumName);
exit;
end;
if PickTreeExec(Address) then
begin
AnalyzeMusicPage(Address); // Album page
end
else
LogMessage('No album selected');
end;
//------------------------------------------------------------------------------
// ANALYZE MUSIC PAGE
//------------------------------------------------------------------------------
procedure AnalyzeMusicPage(Address: string);
var
urlmusic, cover, Lenght, Pays: String;
Autore, style, Commenti, str_min, campo_note: String;
minu, minuti, ore, lgth_comm, save_lgth_comm, i: integer;
j: Real;
begin
Page := GetPage(Address);
Page := UTF8decode(Page);
if debug_album then
DumpPage(folder+'discogsPageDetail.txt', Page); // debug_album
HTMLdecode(Page);
if debug_album then
DumpPage(folder+'discogsPageDetailHTMLdecode.txt', Page); // debug_album
Readdress := '';
Readdress := textbetween(Page, '<meta property="og:url" content="', '">');
if length(Readdress) > 0 then
Address := Readdress;
if Pos('<div class="head">', Page) = 0 then //inizio dati da estrarre
Begin
LogMessage('Pagina album errata - Error while reading album page');
exit;
end;
SetField(fieldURL, Address);
SetField(fieldDate, DateToStr(Date));
Normalizza_Page(Page);
// Page := UTF8Decode(Page); //fs2015-01-15
albumok := True;
//dati generali
initchar := '<div class="profile">'; //io info generali
endchar := '<div class="section_content">'; //inizio tracklist
Value := TextBetween(Page, initchar, endchar);
if debug_album then
DumpPage(folder+'discogsinfo_generali.txt', Value);
initchar := '<div class="head">Label:</div>';
endchar := '</div>';
label := textbetween(value, initchar, endchar) + endchar;
HtmlRemoveTags(label);
label := StringReplace(label, CRLF, '');
label := fulltrim(label);
// label := StringReplace(label, '', '');
// label := StringReplace(label, '–', '');
label := StringReplace(label, ' ', ' ');
SetField(fieldproducer, label);
label := textbetween(Value, '<span itemprop="name" title="', '"');
SetField(fieldoriginalTitle, label); //artista in original title
initchar := '<span itemprop="name">';
endchar := '</span>';
label := textbetween(Value, initchar, endchar);
label := fulltrim(label);
SetField(fieldtranslatedTitle, label);
initchar := '<div class="head">Format:</div>'; //formato LP/Album/CD/ristampa ecc.
endchar := '</div>';
label := textbetween(Value, initchar, endchar) + endchar;
label := FormatText(label);
label := stringReplace(label, CRLF, '');
label := stringreplace (label, ' ', ' ');
label := stringreplace (label, ' ', ' ');
SetField(fieldvideoformat, label); //formato
initchar := '<div class="profile">';
endchar := '</div>';
label := textbetween(Value, '<div class="head">Country:</div>', endchar) + endchar;
label := FormatText(label);
SetField(fieldcountry, label); //paese
label := textbetween(Value, '<div class="head">Released:</div>', endchar) + endchar;
label := FormatText(label);
SetField(fieldyear, label);
//fs20150728 estrazione data della release
nuovo_anno := getfield(fieldYear); //fs20150728 //anno
if length(nuovo_anno) = 0 then
estrai_data_released;
// end;
label := textbetween(Value, '<div class="head">Genre:</div>', endchar) + endchar;
label := FormatText(label);
style := textbetween(Value, '<div class="head">Style:</div>', endchar) + endchar;
style := FormatText(style);
if length(style) > 0 then
label := label + ' (' + style + ')';
SetField(fieldcategory, label); //genere
//discogs tracklist
initchar := 'id="tracklist" class="section tracklist" data-toggle="tracklist">';
endchar := 'class="section credits toggle_section toggle_section_remember"';
Value := TextBetween(Page, initchar, endchar);
if length(Value) = 0 then
begin
endchar := 'class="section notes toggle_section toggle_section_remember"'; //inizio note;
Value := TextBetween(Page, initchar, endchar);
end;
if length(Value) = 0 then
begin
endchar := 'class="section_content toggle_section_content">'; //inizio note;
Value := TextBetween(Page, initchar, endchar);
end;
if length(Value) = 0 then
begin
endchar := 'section notes'; //inizio note;
Value := TextBetween(Page, initchar, endchar);
end;
initchar := '<td class="tracklist_track_pos">';
endchar := '</table>';
Value := initchar + TextBetween(Value, initchar, endchar) + inizio_e_fine_traccia;
if debug_album then
DumpPage(folder+'discogstracks.txt', Value);
brani := '';
estrazione_tracce;
// value := stringReplace(brani, '–', '');
value := stringreplace(brani, 'Composed By', ' - composed by');
value := stringReplace(value, '*', '');
// value := stringReplace(value, '&', '&');
SetField(fieldDescription, Value);
//*** artists
initchar := 'class="section credits';
endchar := '<!-- /credits -->';
Value := TextBetween(Page, initchar, endchar);
initchar := '<div class="section_content toggle_section_content">';
endchar := '</div>';
value := TextBetween(value, initchar, endchar);
if debug_album then
DumpPage(folder+'discogscredits.txt', Value);
brani := '';
estrazione_credits;
value := brani; //fs2015.01.21
value := stringreplace(value, ' ', ' '); //fs2015.01.21
SetField(fieldActors, value);
//*** note
initchar := 'class="section notes';
endchar := 'class="section m_versions';
value := TextBetween(Page, initchar, endchar);
value := FormatText(value);
initchar := 'class="section notes toggle_section toggle_section_remember"'; //inizio note
endchar := '<!-- /notes -->'; //fine note
Value := TextBetween(Page, initchar, endchar);
initchar := 'class="section_content toggle_section_content">';
endchar := '</div>';
value := TextBetween(value, initchar, endchar);
value := FormatText(value);
campo_note := value;
//*** barcode
initchar := 'class="section barcodes toggle_section'; //inizio note
endchar := '<!-- /barcodes -->'; //fine note
Value := TextBetween(Page, initchar, endchar);
initchar := '<ul class="list_no_style">';
endchar := '</ul>';
value := TextBetween(value, initchar, endchar);
// HTMLRemoveTags(value);
value := FormatText(value);
value := stringReplace(value, CRLFspace, CRLF);
if (value = 'Barcode:') or (value = 'barcode:') then
value := '';
commenti := '';
if length(campo_note) > 0 then
begin
commenti := campo_note;
if length(value) > 0 then
begin
commenti := commenti + CRLF + value;
end
end
else
commenti := value;
// end;
// end;
if length(data_rilascio) > 0 then
commenti := 'Released on ' + data_rilascio + CRLF + commenti;
SetField(fieldComments, commenti);
data_rilascio := '';
//*** cover
cover := TextBetween(Page, 'data-images=', '"full"');
initchar := '"http:';
endchar := '"';
cover := initchar + TextBetween(cover, initchar, endchar);
// cover := discogsUrl + cover;
if CanSetPicture then
GetPicture(cover);
end;
Procedure estrazione_tracce;
var
lgth_track: integer;
Begin
// normalizza_page(Value); //fs2015.01.15
initchar := inizio_e_fine_traccia;
endchar := inizio_e_fine_traccia;
lungo := (Length(initchar) + length(endchar));
Dati_traccia := initchar + TextBetween(Value, initchar, endchar) + endchar;
while Length(Dati_traccia) > Lungo do //esegui se trovato qualcosa
begin
datitrack := '';
track := '';
initchar := inizio_e_fine_traccia;
endchar := '</td>';
datitrack := textbetween(dati_traccia, initchar, endchar) + ' '; //n.d'ordine
lgth_track := length(datitrack);
if lgth_track = 2 then //num.traccia <10
datitrack := '0' + datitrack + ' ';
initchar := '<td class="tracklist_track_artists">';
endchar := '</td>';
track := textbetween(dati_traccia, initchar, endchar); //artista in antologie
HTMLRemoveTags(track);
if length(track) > 0 then
datitrack := datitrack + ' ' + track + ' - ';
initchar := '<span class="tracklist_track_title" itemprop="name">';
endchar := '</span>';
track := textbetween(dati_traccia, initchar, endchar); //titolo brano
datitrack := datitrack + track;
initchar := '<blockquote>';
endchar := '</blockquote>';
track := textbetween(dati_traccia, initchar, endchar);
track := stringReplace(track, '<span class="tracklist_extra_artist_span">', '<span class="tracklist_extra_artist_span"> ');
HTMLRemoveTags(track);
if length(track) > 0 then
datitrack := datitrack + ' (' + track + ')';
initchar := 'class="tracklist_track_duration">';
endchar := '</td>';
track := textbetween(dati_traccia, initchar, endchar); //durata brano
HTMLRemoveTags(track);
track := StringReplace(track, CRLF, '');
track := FullTrim(track);
if length(track) > 0 then
datitrack := datitrack + ' (' + track + ')';
// dati del brano raccolti in datitrack
datitrack := stringReplace(datitrack, '( ', '(');
datitrack := stringReplace(datitrack, '( ', '(');
datitrack := stringReplace(datitrack, 'ritten-', 'ritten ');
datitrack := stringReplace(datitrack, '–', '');
datitrack := stringReplace(datitrack, '*', '');
if length(datitrack) > 3 then
brani := brani + datitrack + CRLF; //fs 2014.11.15
// reimposto l'area su cui loopare
initchar := inizio_e_fine_traccia;
endchar := inizio_e_fine_traccia;
delete(Value, 1, length(Dati_traccia) - length(inizio_e_fine_traccia));
dati_traccia := initchar + TextBetween (Value, initchar, endchar) + endchar; //dati_album contiene i dati dell'album
end;
end;
//fs ********************* fine estrazione tracce *****************************************
Procedure estrazione_credits;
var
lgth_track: integer;
oldtrack: string;
Begin
// normalizza_page(Value); //fs2015.01.15
initchar := inizio_credits;
endchar := fine_credits;
oldtrack := '';
lungo := (Length(initchar) + length(endchar));
Dati_traccia := initchar + TextBetween(Value, initchar, endchar) + endchar;
while Length(Dati_traccia) > Lungo do //esegui se trovato qualcosa
begin
datitrack := '';
track := '';
initchar := '<span class="role">';
endchar := '</span>';
datitrack := textbetween(dati_traccia, initchar, endchar) + ' '; //ruolo
initchar := 'class="rollover_link">';
endchar := '</li>';
track := textbetween(dati_traccia, initchar, endchar);
HTMLRemoveTags(track);
if length(track) = 0 then
begin
initchar := '</span>';
track := textbetween(dati_traccia, initchar, endchar);
end;
track := FormatText(track);
track := StringReplace(track, CRLF, '');
datitrack := datitrack + ': ' + track; //nomi
datitrack := stringReplace(datitrack, '*', ''); //fs2015.01.21
// dati del brano raccolti in datitrack
if track <> oldtrack then
brani := brani + datitrack + CRLF;
oldtrack := track;
// reimposto l'area su cui loopare
initchar := inizio_credits;
endchar := fine_credits;
delete(Value, 1, length(Dati_traccia) - length(inizio_credits));
dati_traccia := initchar + TextBetween (Value, initchar, endchar) + endchar;
end;
end;
//fs ********************* fine nuova estrazione credits *****************************************
Procedure Normalizza_page(Pagina: string); // elimina i crlf, trasforma delimiters maiuscoli in minuscoli
begin
pagina := RegExprSetReplace('<([^>]+)>', pagina, '<\L\1>', true);
CharAbNormal := crlf;
CharNormal := ' ';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '<B';
CharNormal := '<b';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</B';
CharNormal := '</b';
pagina := StringReplace(Value, CharAbNormal, CharNormal);
CharAbNormal := '<FONT';
CharNormal := '<font';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</FONT';
CharNormal := '</font';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '<TR';
CharNormal := '<tr';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</TR';
CharNormal := '</tr';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '<TD';
CharNormal := '<td';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</TD';
CharNormal := '</td';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '<DIV';
CharNormal := '<div';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
CharAbNormal := '</DIV';
CharNormal := '</div';
pagina := StringReplace(pagina, CharAbNormal, CharNormal);
end;
//------------------------------------------------------------------------------
// set show warning (normal mode) or add to log (batch mode)
//------------------------------------------------------------------------------
procedure LogMessage(m: string);
begin
if BatchMode > 0 then
AddToLog('item '+GetField(fieldNumber)+': '+m)
else
ShowWarning(m);
end;
//------------------------------------------------------------------------------
// add a message in the batch log and save to disk
// (because I don't know when it's finished...)
//------------------------------------------------------------------------------
procedure AddToLog(m: string);
begin
batchlog.Add(m);
batchlog.SaveToFile(batchlogfic);
end;
//------------------------------------------------------------------------------
// process batch mode
//------------------------------------------------------------------------------
procedure discogsBatch;
begin
AlbumName := GetField(fieldUrl); // if no url or another site then ignore
if (AlbumName <> '') and (Pos(AllAlbumUrl, AlbumName) > 0) then
AnalyzeMusicPage(AlbumName)
else
LogMessage('ignored url="'+AlbumName+'"');
end;
procedure estrai_data_released;
begin
// ShowMessage('data di rilascio ' + nuovo_anno);
label := StringReplace(label, ' ', '/');
data_rilascio := label;
mese_anno := Textafter(data_rilascio, '/');
solo_anno := Textafter(mese_anno, '/');
//***************************
// if length(campo_note) > 0 then
// begin
// commenti := campo_note;
// if length(value) > 0 then
// begin
// commenti := commenti + CRLF + value;
// end
// end
// else
// commenti := value;
//***************************
if length(solo_anno) > 0 then
begin
setfield(fieldyear, solo_anno);
end
else
setfield(fieldyear, solo_anno);
end;
//------------------------------------------------------------------------------
// process normal mode
//------------------------------------------------------------------------------
procedure discogsNorm;
begin
albumok := False;
AlbumName := GetField(fieldTranslatedTitle); // get album name
ArtistName := getField(fieldOriginalTitle); // get artist
confbatch := TStringList.Create;
confbatch.Add('You have selected the normal mode:');
confbatch.Add('if You supply artist name in Original Title field,');
confbatch.Add('I will search discogs database using both');
confbatch.Add('artist and album name.');
confbatch.Add('Results will be more accurate!');
confbatch.Add('');
confbatch.Add('confirm your choice');
if not ShowWarning(confbatch.Text) then
begin
AddToLog('batch mode aborted'); // batch mode not confirmed = abort
abort := 'o';
exit;
end;
repeat
// if not Input('discogs - ' + Value, 'Enter the name of the album :' + crlf, AlbumName) or (AlbumName = '') then exit;
if not Input('discogs = ' + ArtistName + ' - ' + AlbumName, 'Enter the name of the album :'
+ crlf, AlbumName) or (AlbumName = '') then exit;
formato := getfield(fieldmediatype);
if pos(formato, 'cd') > 0 then
formato:= 'CD';
artista := Artistname;
GetList;
until albumok;
end;
//------------------------------------------------------------------------------
// start here
//------------------------------------------------------------------------------
begin
if abort = 'o' then exit; // batch mode aborted
if firstcall <> 'done' then
begin // 1st call: init parameters
firstcall := 'done';
if not CheckVersion(4,2,0) then
begin
ShowMessage('This script requires a newer version of Ant Movie Catalog (at least the version 4.2.0)');
abort := 'o';
exit;
end;
// get user's parms (used more than once)
BatchMode := GetOption('Mode');
if BatchMode > 0 then // batch mode: confirm the choice
begin
batchlog := TStringList.Create; // init batch log
batchlog.Add('starting batch mode');
batchlog.Add(StringOfChar('*',80));
batchlog.SaveToFile(batchlogfic);
// confirmation message
confbatch := TStringList.Create;
confbatch.Add('You have selected the batch mode:');
confbatch.Add('Did you save your database?');
confbatch.Add('*** Address to extract must be supplied in Field Url ***');
confbatch.Add('');
confbatch.Add('At the end of treatement:');
confbatch.Add('- look at the file ' + batchlogfic + ' for errors/infos');
confbatch.Add('- the albums found will be checked, the others not (for the selection)');
confbatch.Add(' (see: Tools/Preferences/Movie list/checkboxes)');
confbatch.Add('');
confbatch.Add('confirm your choice');
if not ShowWarning(confbatch.Text) then
begin
AddToLog('batch mode aborted'); // batch mode not confirmed = abort
abort := 'o';
exit;
end;
end;
end;
// let's go
if BatchMode = 0 then
discogsNorm
else
discogsBatch;
end.