diff --git a/context/lyrics_providers.xml b/context/lyrics_providers.xml index a82171936..6d4eefc67 100644 --- a/context/lyrics_providers.xml +++ b/context/lyrics_providers.xml @@ -23,42 +23,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -77,27 +41,6 @@ - - - - - - - - - - - - @@ -128,10 +71,20 @@ + + + + + + @@ -144,16 +97,6 @@ - - - - - - - - - - @@ -170,29 +113,11 @@ - - - - - - + + - + - - - - @@ -202,28 +127,29 @@ - + + - + - - + + - - - - - - - + - - + - + + + + + @@ -233,47 +159,15 @@ - - - - - - - - - - - - - - - - - - - - + + + + + @@ -285,6 +179,13 @@ + + + + + @@ -318,13 +219,6 @@ - - - - - - - diff --git a/context/ultimatelyrics.cpp b/context/ultimatelyrics.cpp index 8c98514de..6c35cd272 100644 --- a/context/ultimatelyrics.cpp +++ b/context/ultimatelyrics.cpp @@ -61,10 +61,22 @@ static UltimateLyricsProvider::Rule parseRule(QXmlStreamReader* reader) if (QLatin1String("item") == reader->name()) { QXmlStreamAttributes attr = reader->attributes(); if (attr.hasAttribute("tag")) { - ret << UltimateLyricsProvider::RuleItem(attr.value("tag").toString(), QString()); + ret << UltimateLyricsProvider::RuleItem(UltimateLyricsProvider::RuleItem::XmlTag, attr.value("tag").toString()); } else if (attr.hasAttribute("begin")) { - ret << UltimateLyricsProvider::RuleItem(attr.value("begin").toString(), attr.value("end").toString()); + // An item with no 'end' used to be indistinguishable from a 'tag' item, and was + // handled as one - keep that behaviour for hand-written provider files. + ret << (attr.hasAttribute("end") + ? UltimateLyricsProvider::RuleItem(UltimateLyricsProvider::RuleItem::Range, + attr.value("begin").toString(), attr.value("end").toString()) + : UltimateLyricsProvider::RuleItem(UltimateLyricsProvider::RuleItem::XmlTag, + attr.value("begin").toString())); + } + else if (attr.hasAttribute("container")) { + ret << UltimateLyricsProvider::RuleItem(UltimateLyricsProvider::RuleItem::Container, attr.value("container").toString()); + } + else if (attr.hasAttribute("unescape")) { + ret << UltimateLyricsProvider::RuleItem(UltimateLyricsProvider::RuleItem::Unescape, attr.value("unescape").toString()); } } reader->skipCurrentElement(); diff --git a/context/ultimatelyricsprovider.cpp b/context/ultimatelyricsprovider.cpp index 91b832ea9..ce1f31e1e 100644 --- a/context/ultimatelyricsprovider.cpp +++ b/context/ultimatelyricsprovider.cpp @@ -121,7 +121,7 @@ static QString extract(const QString& source, const QString& begin, const QStrin } DBUG << "Found match"; - return source.mid(beginIdx, endIdx - beginIdx - 1); + return source.mid(beginIdx, endIdx - beginIdx); } static QRegularExpression xmlTagRegex = QRegularExpression("<(\\w+).*>"); @@ -138,6 +138,156 @@ static QString extractXmlTag(const QString& source, const QString& tag) return extract(source, tag, "", true); } +// Decode a JSON string literal. Sites that render client-side (e.g. Musixmatch) embed the +// lyrics in a JSON blob rather than in markup, so what the extract rules pull out still has +// its escapes intact. +static QString jsonUnescape(const QString& source) +{ + QString ret; + ret.reserve(source.length()); + + for (int i = 0; i < source.length(); ++i) { + if (QLatin1Char('\\') != source.at(i) || i + 1 >= source.length()) { + ret += source.at(i); + continue; + } + + QChar esc = source.at(++i); + switch (esc.toLatin1()) { + case 'n': ret += QLatin1Char('\n'); break; + case 't': ret += QLatin1Char('\t'); break; + case 'r': ret += QLatin1Char('\r'); break; + case 'b': ret += QLatin1Char('\b'); break; + case 'f': ret += QLatin1Char('\f'); break; + case 'u': { + bool ok = false; + ushort code = i + 4 < source.length() ? source.mid(i + 1, 4).toUShort(&ok, 16) : 0; + if (ok) { + ret += QChar(code); + i += 4; + } + else { + ret += esc; + } + break; + } + // Covers \" \\ \/ - and leaves anything unrecognised as-is. + default: ret += esc; break; + } + } + return ret; +} + +// Matches the opening tag of an element carrying the named attribute. The attribute must be a +// whitespace-separated name followed by '=', so that a name is not matched by a longer one that +// merely starts with it - a word boundary would not do, as '-' ends a word. +static QRegularExpression containerRegex(const QString& attribute) +{ + return QRegularExpression("<(\\w+)\\b[^>]*\\s" + QRegularExpression::escape(attribute) + "\\s*=[^>]*>", + QRegularExpression::CaseInsensitiveOption); +} + +// Given the opening tag matched by containerRegex(), walks the same tag in and out until the +// element's own closing tag is reached, so that nested markup does not terminate it early. +// Sets contentEnd to the start of that closing tag, and elementEnd to just past it. +static bool findContainerEnd(const QString& source, const QRegularExpressionMatch& open, int& contentEnd, int& elementEnd) +{ + QRegularExpression nestRegex("<(/?)" + open.captured(1) + "\\b[^>]*>", QRegularExpression::CaseInsensitiveOption); + int scan = open.capturedEnd(); + int depth = 1; + + while (depth > 0) { + auto nest = nestRegex.match(source, scan); + if (!nest.hasMatch()) { + DBUG << "Failed to find end of container"; + return false; + } + if (nest.captured(1).isEmpty()) { + ++depth; + } + else if (0 == --depth) { + contentEnd = nest.capturedStart(); + elementEnd = nest.capturedEnd(); + } + scan = nest.capturedEnd(); + } + return true; +} + +// Extract the contents of every element carrying the named attribute. Used for sites (e.g. +// Genius) that split the lyrics over several containers whose class names are build-specific, +// and whose contents hold nested markup - so neither a literal tag match nor a naive begin/end +// pair works. +static QString extractContainers(const QString& source, const QString& attribute) +{ + DBUG << "Looking for containers with" << attribute; + QRegularExpression openRegex = containerRegex(attribute); + QStringList parts; + int pos = 0; + + while (pos < source.length()) { + auto open = openRegex.match(source, pos); + if (!open.hasMatch()) { + break; + } + + int contentEnd = -1; + int elementEnd = -1; + if (!findContainerEnd(source, open, contentEnd, elementEnd)) { + break; + } + + // Empty containers are used as placeholders around ad slots - skip them, so that + // joining does not introduce stray breaks between the sections that do have lyrics. + QString content = source.mid(open.capturedEnd(), contentEnd - open.capturedEnd()); + if (!content.trimmed().isEmpty()) { + parts << content; + } + pos = elementEnd; + } + + if (parts.isEmpty()) { + DBUG << "Failed to find container"; + return QString(); + } + + DBUG << "Found" << parts.length() << "container(s)"; + return parts.join(QLatin1String("
")); +} + +// Remove every element carrying the named attribute, contents included. Genius marks the +// page furniture it embeds within the lyrics containers (contributor count, song title, +// summary) with such an attribute, and it has to go along with the markup it sits in. +static QString excludeContainers(const QString& source, const QString& attribute) +{ + DBUG << "Looking for containers with" << attribute; + QRegularExpression openRegex = containerRegex(attribute); + QString ret = source; + int removed = 0; + int pos = 0; + + while (pos < ret.length()) { + auto open = openRegex.match(ret, pos); + if (!open.hasMatch()) { + break; + } + + int contentEnd = -1; + int elementEnd = -1; + if (!findContainerEnd(ret, open, contentEnd, elementEnd)) { + break; + } + + // Any nested match is removed along with this one, so resume from where it started. + ret.remove(open.capturedStart(), elementEnd - open.capturedStart()); + pos = open.capturedStart(); + ++removed; + } + + DBUG << "Removed" << removed << "container(s)"; + return ret; +} + static QString exclude(const QString& source, const QString& begin, const QString& end) { int beginIdx = source.indexOf(begin, 0, Qt::CaseInsensitive); @@ -166,11 +316,21 @@ static QString excludeXmlTag(const QString& source, const QString& tag) static void applyExtractRule(const UltimateLyricsProvider::Rule& rule, QString& content, const Song& song) { for (const UltimateLyricsProvider::RuleItem& item : rule) { - if (item.second.isNull()) { - content = extractXmlTag(content, doTagReplace(item.first, song)); - } - else { - content = extract(content, doTagReplace(item.first, song), doTagReplace(item.second, song)); + switch (item.type) { + case UltimateLyricsProvider::RuleItem::XmlTag: + content = extractXmlTag(content, doTagReplace(item.begin, song)); + break; + case UltimateLyricsProvider::RuleItem::Range: + content = extract(content, doTagReplace(item.begin, song), doTagReplace(item.end, song)); + break; + case UltimateLyricsProvider::RuleItem::Container: + content = extractContainers(content, doTagReplace(item.begin, song)); + break; + case UltimateLyricsProvider::RuleItem::Unescape: + if (QLatin1String("json") == item.begin) { + content = jsonUnescape(content); + } + break; } } } @@ -178,11 +338,19 @@ static void applyExtractRule(const UltimateLyricsProvider::Rule& rule, QString& static void applyExcludeRule(const UltimateLyricsProvider::Rule& rule, QString& content, const Song& song) { for (const UltimateLyricsProvider::RuleItem& item : rule) { - if (item.second.isNull()) { - content = excludeXmlTag(content, doTagReplace(item.first, song)); - } - else { - content = exclude(content, doTagReplace(item.first, song), doTagReplace(item.second, song)); + switch (item.type) { + case UltimateLyricsProvider::RuleItem::XmlTag: + content = excludeXmlTag(content, doTagReplace(item.begin, song)); + break; + case UltimateLyricsProvider::RuleItem::Range: + content = exclude(content, doTagReplace(item.begin, song), doTagReplace(item.end, song)); + break; + case UltimateLyricsProvider::RuleItem::Container: + content = excludeContainers(content, doTagReplace(item.begin, song)); + break; + case UltimateLyricsProvider::RuleItem::Unescape: + // Not meaningful as an exclusion - ignored. + break; } } } @@ -235,22 +403,6 @@ void UltimateLyricsProvider::fetchInfo(int id, Song metadata, bool removeThe) artistFixed = artistFixed.mid(constThe.length()); } - if (QLatin1String("lyrics.wikia.com") == name) { - QUrl url(urlText); - QUrlQuery query; - - query.addQueryItem(QLatin1String("artist"), artistFixed); - query.addQueryItem(QLatin1String("song"), titleFixed); - query.addQueryItem(QLatin1String("func"), QLatin1String("getSong")); - query.addQueryItem(QLatin1String("fmt"), QLatin1String("xml")); - url.setQuery(query); - - NetworkJob* reply = NetworkAccessManager::self()->get(url); - requests[reply] = id; - connect(reply, SIGNAL(finished()), this, SLOT(wikiMediaSearchResponse())); - return; - } - metadata.priority = removeThe ? 1 : 0;// HACK Use this to indicate if searching without 'The ' songs.insert(id, metadata); diff --git a/context/ultimatelyricsprovider.h b/context/ultimatelyricsprovider.h index 80378dbb9..05853e28f 100644 --- a/context/ultimatelyricsprovider.h +++ b/context/ultimatelyricsprovider.h @@ -42,7 +42,21 @@ class UltimateLyricsProvider : public QObject { UltimateLyricsProvider(); ~UltimateLyricsProvider() override; - typedef QPair RuleItem; + struct RuleItem { + enum Type { + XmlTag, // + Range, // + Container, // + Unescape // + }; + + RuleItem(Type t, const QString& b, const QString& e = QString()) + : type(t), begin(b), end(e) {} + + Type type; + QString begin; + QString end; + }; typedef QList Rule; typedef QPair UrlFormat; diff --git a/gui/settings.cpp b/gui/settings.cpp index 76ba25eee..ed59f6ce1 100644 --- a/gui/settings.cpp +++ b/gui/settings.cpp @@ -340,9 +340,7 @@ MPDParseUtils::CueSupport Settings::cueSupport() QStringList Settings::lyricProviders() { - return cfg.get("lyricProviders", QStringList() << "azlyrics.com" - << "chartlyrics.com" - << "lyrics.wikia.com"); + return cfg.get("lyricProviders", QStringList() << "genius.com"); } QStringList Settings::wikipediaLangs()