Well, after so many attempts with claude and deepseek I managed to get a working code for the awards, the problem? only get the first categories shown in the award page, is unable to extract the hidden rows.
Code: Select all
procedure ImportAwards(fieldName: integer);
var
PageText, JsonData, Value, FieldValue, ValuesDelimiter: string;
EventName, CategoryText, RowTitle, PersonNames, CurrentEventName: string;
TempStr, EventBlock, TempStr2: string;
i: integer;
TempText: string; // Temporary variable for UTF-8 fix
begin
PageText := GetPage(MovieUrl + '/awards');
ValuesDelimiter := #13#10;
if (CanSetField(fieldName)) then
begin
FieldValue := '';
CurrentEventName := '';
if (PageText = '') then
Exit;
// Extract JSON from __NEXT_DATA__ script tag
JsonData := TextBetween(PageText, '<script id="__NEXT_DATA__" type="application/json">', '</script>');
if (JsonData = '') then
Exit;
// Check if there are any awards at all
if (Pos('"categories"', JsonData) = 0) then
Exit;
// Start from categories section
Value := TextAfter(JsonData, '"categories":[');
i := 0;
// Process each event
while (Pos('"name":"', Value) > 0) and (i < 200) do
begin
i := i + 1;
// Extract event name
EventName := TextBetween(Value, '"name":"', '"');
// Skip if empty or too long
if (EventName = '') or (Length(EventName) < 3) or (Length(EventName) > 150) then
begin
Value := TextAfter(Value, '"name":"');
Continue;
end;
// Extract the entire event block to process all its awards
EventBlock := TextBetween(Value, '"name":"' + EventName + '"', '"name":"');
if (EventBlock = '') then
EventBlock := Copy(Value, Pos('"name":"' + EventName + '"', Value), Length(Value));
// Check if this event has awards
if (Pos('"items":', EventBlock) > 0) and (Pos('"awardCategoryName"', EventBlock) > 0) then
begin
// This is an award event!
if (EventName <> CurrentEventName) then
begin
// Add event header
if (FieldValue <> '') then
FieldValue := FieldValue + #13#10 + #13#10;
// Fix UTF-8 characters inline - use simple string replacement
TempText := CleanHtml(EventName);
TempText := StringReplace(TempText, 'é', 'é');
TempText := StringReplace(TempText, 'è', 'è');
TempText := StringReplace(TempText, 'ê', 'ê');
TempText := StringReplace(TempText, 'ë', 'ë');
TempText := StringReplace(TempText, 'á', 'á');
TempText := StringReplace(TempText, 'Ã ', 'à');
TempText := StringReplace(TempText, 'â', 'â');
TempText := StringReplace(TempText, 'ã', 'ã');
TempText := StringReplace(TempText, 'ä', 'ä');
TempText := StringReplace(TempText, 'ñ', 'ñ');
TempText := StringReplace(TempText, 'ó', 'ó');
TempText := StringReplace(TempText, 'ò', 'ò');
TempText := StringReplace(TempText, 'ö', 'ö');
TempText := StringReplace(TempText, 'ú', 'ú');
TempText := StringReplace(TempText, 'ü', 'ü');
TempText := StringReplace(TempText, 'ç', 'ç');
// Add the missing replacements for the remaining issues
TempText := StringReplace(TempText, 'Ã', 'í');
TempText := StringReplace(TempText, 'æ', 'æ');
TempText := StringReplace(TempText, 'ø', 'ø');
TempText := StringReplace(TempText, 'Ã…', 'Å');
TempText := StringReplace(TempText, 'Ø', 'Ø');
TempText := StringReplace(TempText, 'Æ', 'Æ');
TempText := StringReplace(TempText, 'þ', 'þ');
TempText := StringReplace(TempText, 'ÿ', 'ÿ');
TempText := StringReplace(TempText, 'ß', 'ß');
TempText := StringReplace(TempText, '–', '–');
TempText := StringReplace(TempText, '—', '—');
TempText := StringReplace(TempText, '•', '•');
TempText := StringReplace(TempText, 'â„¢', '™');
TempText := StringReplace(TempText, '…', '…');
FieldValue := FieldValue + TempText;
CurrentEventName := EventName;
end;
// Process all awards in this event block
TempStr := EventBlock;
while (Pos('"awardCategoryName"', TempStr) > 0) do
begin
TempStr := TextAfter(TempStr, '"awardCategoryName"');
CategoryText := TextBetween(TempStr, '"text":"', '"');
// Get rowTitle (contains Winner/Nominee)
RowTitle := TextBetween(TempStr, '"rowTitle":"', '"');
if (CategoryText <> '') and (Length(CategoryText) > 2) and (Length(CategoryText) < 200) then
begin
FieldValue := FieldValue + #13#10;
// Add Winner/Nominee prefix
if (Pos('Winner', RowTitle) > 0) then
FieldValue := FieldValue + 'Winner: '
else if (Pos('Nominee', RowTitle) > 0) then
FieldValue := FieldValue + 'Nominee: ';
// Fix UTF-8 characters for CategoryText
TempText := CleanHtml(CategoryText);
TempText := StringReplace(TempText, 'é', 'é');
TempText := StringReplace(TempText, 'è', 'è');
TempText := StringReplace(TempText, 'ê', 'ê');
TempText := StringReplace(TempText, 'ë', 'ë');
TempText := StringReplace(TempText, 'á', 'á');
TempText := StringReplace(TempText, 'Ã ', 'à');
TempText := StringReplace(TempText, 'â', 'â');
TempText := StringReplace(TempText, 'ã', 'ã');
TempText := StringReplace(TempText, 'ä', 'ä');
TempText := StringReplace(TempText, 'ñ', 'ñ');
TempText := StringReplace(TempText, 'ó', 'ó');
TempText := StringReplace(TempText, 'ò', 'ò');
TempText := StringReplace(TempText, 'ö', 'ö');
TempText := StringReplace(TempText, 'ú', 'ú');
TempText := StringReplace(TempText, 'ü', 'ü');
TempText := StringReplace(TempText, 'ç', 'ç');
// Add the missing replacements for the remaining issues
TempText := StringReplace(TempText, 'Ã', 'í');
TempText := StringReplace(TempText, 'æ', 'æ');
TempText := StringReplace(TempText, 'ø', 'ø');
TempText := StringReplace(TempText, 'Ã…', 'Å');
TempText := StringReplace(TempText, 'Ø', 'Ø');
TempText := StringReplace(TempText, 'Æ', 'Æ');
TempText := StringReplace(TempText, 'þ', 'þ');
TempText := StringReplace(TempText, 'ÿ', 'ÿ');
TempText := StringReplace(TempText, 'ß', 'ß');
TempText := StringReplace(TempText, '–', '–');
TempText := StringReplace(TempText, '—', '—');
TempText := StringReplace(TempText, '•', '•');
TempText := StringReplace(TempText, 'â„¢', '™');
TempText := StringReplace(TempText, '…', '…');
FieldValue := FieldValue + TempText;
// Extract person names
PersonNames := '';
if (Pos('"subListContent"', TempStr) > 0) then
begin
TempStr2 := TextBetween(Copy(TempStr, 1, 2000), '"subListContent":[', ']');
if (TempStr2 <> '') then
begin
// Extract all names from the array
while (Pos('"text":"', TempStr2) > 0) do
begin
if (PersonNames <> '') then
PersonNames := PersonNames + ', ';
// Fix UTF-8 characters for person names
TempText := CleanHtml(TextBetween(TempStr2, '"text":"', '"'));
TempText := StringReplace(TempText, 'é', 'é');
TempText := StringReplace(TempText, 'è', 'è');
TempText := StringReplace(TempText, 'ê', 'ê');
TempText := StringReplace(TempText, 'ë', 'ë');
TempText := StringReplace(TempText, 'á', 'á');
TempText := StringReplace(TempText, 'Ã ', 'à');
TempText := StringReplace(TempText, 'â', 'â');
TempText := StringReplace(TempText, 'ã', 'ã');
TempText := StringReplace(TempText, 'ä', 'ä');
TempText := StringReplace(TempText, 'ñ', 'ñ');
TempText := StringReplace(TempText, 'ó', 'ó');
TempText := StringReplace(TempText, 'ò', 'ò');
TempText := StringReplace(TempText, 'ö', 'ö');
TempText := StringReplace(TempText, 'ú', 'ú');
TempText := StringReplace(TempText, 'ü', 'ü');
TempText := StringReplace(TempText, 'ç', 'ç');
// Add the missing replacements for the remaining issues
TempText := StringReplace(TempText, 'Ã', 'í');
TempText := StringReplace(TempText, 'æ', 'æ');
TempText := StringReplace(TempText, 'ø', 'ø');
TempText := StringReplace(TempText, 'Ã…', 'Å');
TempText := StringReplace(TempText, 'Ø', 'Ø');
TempText := StringReplace(TempText, 'Æ', 'Æ');
TempText := StringReplace(TempText, 'þ', 'þ');
TempText := StringReplace(TempText, 'ÿ', 'ÿ');
TempText := StringReplace(TempText, 'ß', 'ß');
TempText := StringReplace(TempText, '–', '–');
TempText := StringReplace(TempText, '—', '—');
TempText := StringReplace(TempText, '•', '•');
TempText := StringReplace(TempText, 'â„¢', '™');
TempText := StringReplace(TempText, '…', '…');
PersonNames := PersonNames + TempText;
TempStr2 := TextAfter(TempStr2, '"text":"');
end;
end;
end
else if (Pos('"caption":"', TempStr) > 0) then
begin
PersonNames := TextBetween(Copy(TempStr, 1, 800), '"caption":"', '"');
if (PersonNames <> '') and (Length(PersonNames) < 100) then
begin
// Fix UTF-8 characters for person names
TempText := CleanHtml(PersonNames);
TempText := StringReplace(TempText, 'é', 'é');
TempText := StringReplace(TempText, 'è', 'è');
TempText := StringReplace(TempText, 'ê', 'ê');
TempText := StringReplace(TempText, 'ë', 'ë');
TempText := StringReplace(TempText, 'á', 'á');
TempText := StringReplace(TempText, 'Ã ', 'à');
TempText := StringReplace(TempText, 'â', 'â');
TempText := StringReplace(TempText, 'ã', 'ã');
TempText := StringReplace(TempText, 'ä', 'ä');
TempText := StringReplace(TempText, 'ñ', 'ñ');
TempText := StringReplace(TempText, 'ó', 'ó');
TempText := StringReplace(TempText, 'ò', 'ò');
TempText := StringReplace(TempText, 'ö', 'ö');
TempText := StringReplace(TempText, 'ú', 'ú');
TempText := StringReplace(TempText, 'ü', 'ü');
TempText := StringReplace(TempText, 'ç', 'ç');
// Add the missing replacements for the remaining issues
TempText := StringReplace(TempText, 'Ã', 'í');
TempText := StringReplace(TempText, 'æ', 'æ');
TempText := StringReplace(TempText, 'ø', 'ø');
TempText := StringReplace(TempText, 'Ã…', 'Å');
TempText := StringReplace(TempText, 'Ø', 'Ø');
TempText := StringReplace(TempText, 'Æ', 'Æ');
TempText := StringReplace(TempText, 'þ', 'þ');
TempText := StringReplace(TempText, 'ÿ', 'ÿ');
TempText := StringReplace(TempText, 'ß', 'ß');
TempText := StringReplace(TempText, '–', '–');
TempText := StringReplace(TempText, '—', '—');
TempText := StringReplace(TempText, '•', '•');
TempText := StringReplace(TempText, 'â„¢', '™');
TempText := StringReplace(TempText, '…', '…');
PersonNames := TempText;
end
else
PersonNames := '';
end;
if (PersonNames <> '') then
FieldValue := FieldValue + ' - ' + PersonNames;
end;
end;
end;
// Move to next event
if (EventBlock <> '') and (Length(EventBlock) < Length(Value)) then
Value := TextAfter(Value, EventBlock)
else
Value := TextAfter(Value, '"name":"' + EventName + '"');
end;
// Save the field if we found awards
if (FieldValue <> '') then
begin
if (GetField(fieldName) <> '') then
SetField(fieldName, GetField(fieldName) + #13#10 + #13#10 + 'AWARDS:' + #13#10 + FieldValue)
else
SetField(fieldName, 'AWARDS:' + #13#10 + FieldValue);
end;
end;
end;