@@ -60,6 +64,7 @@
const QLatin1String SongView::constLyricsDir("lyrics/");
const QLatin1String SongView::constExtension(".lyrics");
+const QLatin1String SongView::constSourceExt(".lyrics.json");
const QLatin1String SongView::constCacheDir("tracks/");
const QLatin1String SongView::constInfoExt(".html.gz");
@@ -81,6 +86,17 @@ static QString lyricsCacheFileName(const Song& song, bool createDir = false)
return dir + Covers::encodeName(song.basicTitle()) + SongView::constExtension;
}
+// Sits next to the cached lyrics, and records where they came from. Deliberately never written to
+// the MPD music dir - it is shared with other clients, which have no use for a Cantata specific file.
+static QString sourceInfoFileName(const Song& song, bool createDir = false)
+{
+ QString dir = Utils::cacheDir(SongView::constLyricsDir + Covers::encodeName(song.basicArtist()) + Utils::constDirSep, createDir);
+ if (dir.isEmpty()) {
+ return QString();
+ }
+ return dir + Covers::encodeName(song.basicTitle()) + SongView::constSourceExt;
+}
+
#if !defined Q_OS_WIN && !defined Q_OS_MAC
static QString lyricsOtherFileName(const Song& song)
{
@@ -99,7 +115,7 @@ static inline QString fixNewLines(const QString& o)
}
SongView::SongView(QWidget* p)
- : View(p, QStringList() << tr("Lyrics") << tr("Information") << tr("Metadata")), scrollTimer(nullptr), songPos(0), currentProvider(-1), currentRequest(0), mode(Mode_Display), job(nullptr), currentProv(nullptr), lyricsNeedsUpdating(true), infoNeedsUpdating(true), metadataNeedsUpdating(true)
+ : View(p, QStringList() << tr("Lyrics") << tr("Information") << tr("Metadata")), scrollTimer(nullptr), songPos(0), currentProvider(-1), currentRequest(0), mode(Mode_Display), sourceProvidesMarkup(false), job(nullptr), currentProv(nullptr), lyricsNeedsUpdating(true), infoNeedsUpdating(true), metadataNeedsUpdating(true)
{
scrollAction = ActionCollection::get()->createAction("scrolllyrics", tr("Scroll Lyrics"), Icons::self()->downIcon);
refreshAction = ActionCollection::get()->createAction("refreshlyrics", tr("Refresh Lyrics"), Icons::self()->refreshIcon);
@@ -112,7 +128,7 @@ SongView::SongView(QWidget* p)
connect(refreshAction, SIGNAL(triggered()), SLOT(update()));
connect(editAction, SIGNAL(triggered()), SLOT(edit()));
connect(delAction, SIGNAL(triggered()), SLOT(del()));
- connect(UltimateLyrics::self(), SIGNAL(lyricsReady(int, QString)), SLOT(lyricsReady(int, QString)));
+ connect(UltimateLyrics::self(), SIGNAL(lyricsReady(int, QString, QUrl)), SLOT(lyricsReady(int, QString, QUrl)));
engine = ContextEngine::create(this);
refreshInfoAction = ActionCollection::get()->createAction("refreshtrack", tr("Refresh Track Information"), Icons::self()->refreshIcon);
@@ -161,6 +177,7 @@ void SongView::update()
if (cacheExists) {
QFile::remove(cacheName);
}
+ QFile::remove(sourceInfoFileName(currentSong));
break;
default:
return;
@@ -193,6 +210,7 @@ void SongView::search()
if (!cacheName.isEmpty() && QFile::exists(cacheName)) {
QFile::remove(cacheName);
}
+ QFile::remove(sourceInfoFileName(currentSong));
update(dlg.song(), true);
}
}
@@ -226,6 +244,7 @@ void SongView::del()
if (!mpdName.isEmpty() && QFile::exists(mpdName)) {
QFile::remove(mpdName);
}
+ QFile::remove(sourceInfoFileName(currentSong));
}
void SongView::showContextMenu(const QPoint& pos)
@@ -297,6 +316,12 @@ void SongView::songPosition()
void SongView::scroll()
{
+ // An open annotation both moves the lyrics down and is something the reader is in the middle
+ // of, so leave the view where they put it until they close it again.
+ if (!expandedAnnotations.isEmpty()) {
+ return;
+ }
+
if (Mode_Display == mode && scrollAction->isChecked() && scrollTimer) {
QScrollBar* bar = text->verticalScrollBar();
@@ -378,14 +403,19 @@ void SongView::loadLyricsFromFile()
QString tagLyrics = Tags::readLyrics(songFile);
if (!tagLyrics.isEmpty()) {
- text->setText(fixNewLines(tagLyrics));
+ lyricsPlain = tagLyrics;
+ lyricsSource = QUrl();
+ lyricsSourceName = QString();
+ sourceProvidesMarkup = false;
+ clearAnnotations();
+ renderLyrics();
setMode(Mode_Display);
// controls->setVisible(false);
return;
}
#endif
// Stop here if we found lyrics in the cache dir.
- if (setLyricsFromFile(mpdLyrics)) {
+ if (setLyricsFromFile(mpdLyrics, true)) {
lyricsFile = mpdLyrics;
setMode(Mode_Display);
return;
@@ -402,7 +432,7 @@ void SongView::loadLyricsFromFile()
// Check for cached file...
QString file = lyricsCacheFileName(currentSong);
- if (setLyricsFromFile(file)) {
+ if (setLyricsFromFile(file, true)) {
// We just wanted a normal update without explicit re-fetching. We can return
// here because we got cached lyrics and we don't want an explicit re-fetch.
lyricsFile = file;
@@ -704,6 +734,13 @@ void SongView::abortInfoSearch()
void SongView::showMoreInfo(const QUrl& url)
{
+ QUrlQuery query(url);
+
+ if (QLatin1String("cantata") == url.scheme() && query.hasQueryItem(LyricsMarkup::constAnnotationQuery)) {
+ toggleAnnotation(query.queryItemValue(LyricsMarkup::constAnnotationQuery));
+ return;
+ }
+
QDesktopServices::openUrl(url);
}
@@ -721,6 +758,11 @@ void SongView::abort()
job = nullptr;
}
currentProvider = -1;
+ lyricsPlain = QString();
+ lyricsSource = QUrl();
+ lyricsSourceName = QString();
+ sourceProvidesMarkup = false;
+ clearAnnotations();
if (currentProv) {
currentProv->abort();
currentProv = nullptr;
@@ -805,7 +847,12 @@ void SongView::downloadFinished()
QTextStream str(reply->actualJob());
QString lyrics = str.readAll();
if (!lyrics.isEmpty()) {
- text->setText(fixNewLines(lyrics));
+ lyricsPlain = lyrics;
+ lyricsSource = QUrl();
+ lyricsSourceName = QString();
+ sourceProvidesMarkup = false;
+ clearAnnotations();
+ renderLyrics();
cancelJobAction->setEnabled(false);
hideSpinner();
return;
@@ -817,7 +864,7 @@ void SongView::downloadFinished()
loadLyricsFromFile();
}
-void SongView::lyricsReady(int id, QString lyrics)
+void SongView::lyricsReady(int id, QString lyrics, QUrl source)
{
if (id != currentRequest) {
return;
@@ -840,16 +887,209 @@ void SongView::lyricsReady(int id, QString lyrics)
getLyrics();
}
else {
+ // Take the lyrics back out of the browser, rather than using 'plain' directly, so that
+ // what we save stays exactly what this round-trip has always produced.
text->setText(fixNewLines(plain));
+ lyricsPlain = text->toPlainText();
lyricsFile = QString();
if (!(Settings::self()->storeLyricsInMpdDir() && !currentSong.isNonMPD() && saveFile(mpdLyricsFilePath(currentSong)))) {
saveFile(lyricsCacheFileName(currentSong, true));
}
+ lyricsSource = source;
+ lyricsSourceName = currentProv ? currentProv->displayName() : QString();
+ sourceProvidesMarkup = currentProv && currentProv->providesMarkup();
+ clearAnnotations();
+ if (sourceProvidesMarkup && Settings::self()->lyricsAnnotations()) {
+ lyricsMarkup = LyricsMarkup::prepare(lyrics);
+ // Keep what the provider sent, so that the annotations are still there next time
+ // these lyrics are read back from the cache.
+ lyricsProviderMarkup = lyricsMarkup.hasAnnotations() ? lyrics : QString();
+ }
+ // After saveFile(), so that the source info is never older than the lyrics it describes.
+ saveSourceInfo();
+ renderLyrics();
setMode(Mode_Display);
}
}
}
+void SongView::renderLyrics(bool keepPosition)
+{
+ QString html = lyricsMarkup.hasAnnotations()
+ ? LyricsMarkup::render(lyricsMarkup, annotationBodies, expandedAnnotations)
+ : fixNewLines(sourceProvidesMarkup ? LyricsMarkup::styleSectionHeaders(lyricsPlain) : lyricsPlain);
+
+ if (lyricsSource.isValid() && !lyricsSourceName.isEmpty()) {
+ QString link = QLatin1String("") + lyricsSourceName.toHtmlEscaped() + QLatin1String("");
+ html += QLatin1String("") + tr("Lyrics from %1").arg(link) + QLatin1String("
");
+ }
+
+ QScrollBar* bar = keepPosition ? text->verticalScrollBar() : nullptr;
+ int position = bar ? bar->value() : 0;
+
+ setHtml(html, Page_Lyrics);
+
+ if (bar) {
+ bar->setValue(position);
+ }
+}
+
+void SongView::toggleAnnotation(const QString& id)
+{
+ if (id.isEmpty()) {
+ return;
+ }
+
+ if (expandedAnnotations.contains(id)) {
+ expandedAnnotations.remove(id);
+ }
+ else {
+ expandedAnnotations.insert(id);
+
+ if (!annotationBodies.contains(id)) {
+ // Only ever fetched when a reader asks for one, so a song with fifty annotations costs
+ // nothing until they are opened.
+ QNetworkRequest req(QUrl(QLatin1String("https://genius.com/api/annotations/") + id + QLatin1String("?text_format=plain")));
+ req.setRawHeader("User-Agent", "Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0");
+ NetworkJob* reply = NetworkAccessManager::self()->get(req, 5000);
+ annotationJobs.insert(reply, id);
+ connect(reply, SIGNAL(finished()), this, SLOT(annotationFetched()));
+ }
+ }
+
+ renderLyrics(true);
+}
+
+void SongView::annotationFetched()
+{
+ NetworkJob* reply = qobject_cast(sender());
+
+ if (!reply) {
+ return;
+ }
+
+ reply->deleteLater();
+ QString id = annotationJobs.take(reply);
+
+ if (id.isEmpty()) {
+ return;
+ }
+
+ QString body;
+
+ if (reply->ok()) {
+ // Asking for the plain text form keeps Genius' own markup - images, embedded links - out of
+ // the document the lyrics live in.
+ body = QJsonDocument::fromJson(reply->readAll())
+ .object()
+ .value(QLatin1String("response"))
+ .toObject()
+ .value(QLatin1String("annotation"))
+ .toObject()
+ .value(QLatin1String("body"))
+ .toObject()
+ .value(QLatin1String("plain"))
+ .toString()
+ .trimmed();
+ }
+
+ annotationBodies.insert(id, body.isEmpty()
+ ? tr("Annotation unavailable.")
+ : body.toHtmlEscaped().replace(QLatin1String("\n"), QLatin1String("
")));
+ renderLyrics(true);
+}
+
+void SongView::clearAnnotations()
+{
+ // Emptied before cancelling, as aborting a reply reports it as finished - which would otherwise
+ // come back through annotationFetched() while we are still tearing down.
+ QList jobs = annotationJobs.keys();
+ annotationJobs.clear();
+ for (NetworkJob* job : jobs) {
+ job->cancelAndDelete();
+ }
+ lyricsProviderMarkup = QString();
+ lyricsMarkup.clear();
+ annotationBodies.clear();
+ expandedAnnotations.clear();
+}
+
+void SongView::saveSourceInfo()
+{
+ QString fileName = sourceInfoFileName(currentSong, true);
+
+ if (fileName.isEmpty()) {
+ return;
+ }
+
+ if (!lyricsSource.isValid() || !currentProv) {
+ QFile::remove(fileName);
+ return;
+ }
+
+ QJsonObject obj;
+ obj.insert(QLatin1String("provider"), currentProv->getName());
+ obj.insert(QLatin1String("url"), lyricsSource.toString());
+ if (!lyricsProviderMarkup.isEmpty()) {
+ obj.insert(QLatin1String("markup"), lyricsProviderMarkup);
+ }
+
+ QFile f(fileName);
+ if (f.open(QIODevice::WriteOnly)) {
+ f.write(QJsonDocument(obj).toJson(QJsonDocument::Compact));
+ }
+}
+
+void SongView::loadSourceInfo(const QString& lyricsFilePath)
+{
+ QString fileName = sourceInfoFileName(currentSong);
+
+ if (fileName.isEmpty()) {
+ return;
+ }
+
+ // The lyrics may have been edited, or replaced by another client, since we recorded where they
+ // came from - in which case what we recorded no longer describes them.
+ QFileInfo sourceInfo(fileName);
+ if (!sourceInfo.exists() || sourceInfo.lastModified() < QFileInfo(lyricsFilePath).lastModified()) {
+ return;
+ }
+
+ QFile f(fileName);
+ if (!f.open(QIODevice::ReadOnly)) {
+ return;
+ }
+
+ QJsonObject obj = QJsonDocument::fromJson(f.readAll()).object();
+ QUrl url(obj.value(QLatin1String("url")).toString());
+
+ if (!url.isValid()) {
+ return;
+ }
+
+ lyricsSource = url;
+ lyricsSourceName = obj.value(QLatin1String("provider")).toString();
+
+ // Prefer how the provider presents itself, as the stored name carries markers - '(PORTUGUESE)'
+ // and the like - that are meant to be translated before being shown. Looking the provider up
+ // also tells us how these lyrics may be rendered, which we cannot ask currentProv about as
+ // nothing was fetched.
+ for (UltimateLyricsProvider* provider : UltimateLyrics::self()->getProviders()) {
+ if (provider->getName() == lyricsSourceName) {
+ lyricsSourceName = provider->displayName();
+ sourceProvidesMarkup = provider->providesMarkup();
+ break;
+ }
+ }
+
+ if (sourceProvidesMarkup && Settings::self()->lyricsAnnotations()) {
+ lyricsProviderMarkup = obj.value(QLatin1String("markup")).toString();
+ if (!lyricsProviderMarkup.isEmpty()) {
+ lyricsMarkup = LyricsMarkup::prepare(lyricsProviderMarkup);
+ }
+ }
+}
+
bool SongView::saveFile(const QString& fileName)
{
QFile f(fileName);
@@ -860,7 +1100,7 @@ bool SongView::saveFile(const QString& fileName)
stream.setEncoding(QStringConverter::Utf8);
stream.setGenerateByteOrderMark(true);
#endif
- stream << text->toPlainText();
+ stream << lyricsPlain;
f.close();
lyricsFile = fileName;
return true;
@@ -892,6 +1132,11 @@ void SongView::getLyrics()
else {
text->setText(QString());
currentProvider = -1;
+ lyricsPlain = QString();
+ lyricsSource = QUrl();
+ lyricsSourceName = QString();
+ sourceProvidesMarkup = false;
+ clearAnnotations();
// Set lyrics file anyway - so that editing is enabled!
lyricsFile = Settings::self()->storeLyricsInMpdDir() && !currentSong.isNonMPD()
? mpdLyricsFilePath(currentSong)
@@ -922,17 +1167,26 @@ void SongView::setMode(Mode m)
}
}
-bool SongView::setLyricsFromFile(const QString& filePath)
+bool SongView::setLyricsFromFile(const QString& filePath, bool useSourceInfo)
{
QFile f(filePath);
if (f.exists() && f.open(QIODevice::ReadOnly)) {
// Read the file using a QTextStream so we get automatic UTF8 detection.
QTextStream inputStream(&f);
- text->setText(fixNewLines(inputStream.readAll()));
+ lyricsPlain = inputStream.readAll();
+ f.close();
+
+ lyricsSource = QUrl();
+ lyricsSourceName = QString();
+ sourceProvidesMarkup = false;
+ clearAnnotations();
+ if (useSourceInfo) {
+ loadSourceInfo(filePath);
+ }
+ renderLyrics();
cancelJobAction->setEnabled(false);
hideSpinner();
- f.close();
return true;
}
diff --git a/context/songview.h b/context/songview.h
index ac53ccb54..a1d1cedb2 100644
--- a/context/songview.h
+++ b/context/songview.h
@@ -25,7 +25,11 @@
#define SONG_VIEW_H
#include "config.h"
+#include "lyricsmarkup.h"
#include "view.h"
+#include
+#include
+#include
#include
class UltimateLyricsProvider;
@@ -52,6 +56,7 @@ class SongView : public View {
public:
static const QLatin1String constLyricsDir;
static const QLatin1String constExtension;
+ static const QLatin1String constSourceExt;
static const QLatin1String constCacheDir;
static const QLatin1String constInfoExt;
@@ -66,7 +71,7 @@ class SongView : public View {
public Q_SLOTS:
void downloadFinished();
- void lyricsReady(int, QString lyrics);
+ void lyricsReady(int, QString lyrics, QUrl source);
void update();
void search();
void edit();
@@ -83,6 +88,7 @@ private Q_SLOTS:
void infoSearchResponse(const QString& resp, const QString& lang);
void abortInfoSearch();
void showMoreInfo(const QUrl& url);
+ void annotationFetched();
private:
void loadLyrics();
@@ -98,15 +104,52 @@ private Q_SLOTS:
void setMode(Mode m);
bool saveFile(const QString& fileName);
+ /**
+ * Builds the lyrics page from lyricsPlain, appending a link to where the lyrics came from
+ * if we know it. All lyrics display goes through here, so that whatever we add to the page
+ * can never end up in the file we save.
+ *
+ * @param keepPosition Whether to leave the view where it is, for a re-render of lyrics the
+ * reader is already part way through. False when showing new lyrics.
+ */
+ void renderLyrics(bool keepPosition = false);
+
+ /**
+ * Records, alongside the cached lyrics, which provider supplied them and the page they can be
+ * read on. The lyrics file itself is left alone - other MPD clients read it too.
+ */
+ void saveSourceInfo();
+
+ /**
+ * Restores what saveSourceInfo() recorded, provided it is no older than the lyrics it
+ * describes.
+ *
+ * @param lyricsFilePath The lyrics file the source information has to match.
+ */
+ void loadSourceInfo(const QString& lyricsFilePath);
+
+ /**
+ * Opens or closes an annotation, fetching its text the first time it is opened.
+ *
+ * @param id The annotation's id, as carried by the link that was clicked.
+ */
+ void toggleAnnotation(const QString& id);
+
+ /** Discards the annotations, along with anything still being fetched for them. */
+ void clearAnnotations();
+
/**
* Reads the lyrics from the given filePath and updates
* the UI with those lyrics.
*
* @param filePath The path to the lyrics file which will be read.
*
+ * @param useSourceInfo Whether the file is one we wrote ourselves, and may therefore have
+ * recorded source information for. False for user supplied files.
+ *
* @return Returns true if the file could be read; otherwise false.
*/
- bool setLyricsFromFile(const QString& filePath);
+ bool setLyricsFromFile(const QString& filePath, bool useSourceInfo = false);
private:
QTimer* scrollTimer;
@@ -119,6 +162,15 @@ private Q_SLOTS:
Action* delAction;
Mode mode;
QString lyricsFile;
+ QString lyricsPlain;
+ QUrl lyricsSource;
+ QString lyricsSourceName;
+ bool sourceProvidesMarkup;
+ QString lyricsProviderMarkup;
+ LyricsMarkup::Prepared lyricsMarkup;
+ QHash annotationBodies;
+ QSet expandedAnnotations;
+ QHash annotationJobs;
QString preEdit;
NetworkJob* job;
UltimateLyricsProvider* currentProv;
diff --git a/context/ultimatelyrics.cpp b/context/ultimatelyrics.cpp
index 8c98514de..423e7fc71 100644
--- a/context/ultimatelyrics.cpp
+++ b/context/ultimatelyrics.cpp
@@ -61,10 +61,22 @@ static UltimateLyricsProvider::Rule parseRule(QXmlStreamReader* reader)
if (QLatin1String("item") == reader->name()) {
QXmlStreamAttributes attr = reader->attributes();
if (attr.hasAttribute("tag")) {
- ret << UltimateLyricsProvider::RuleItem(attr.value("tag").toString(), QString());
+ ret << UltimateLyricsProvider::RuleItem(UltimateLyricsProvider::RuleItem::XmlTag, attr.value("tag").toString());
}
else if (attr.hasAttribute("begin")) {
- ret << UltimateLyricsProvider::RuleItem(attr.value("begin").toString(), attr.value("end").toString());
+ // An item with no 'end' used to be indistinguishable from a 'tag' item, and was
+ // handled as one - keep that behaviour for hand-written provider files.
+ ret << (attr.hasAttribute("end")
+ ? UltimateLyricsProvider::RuleItem(UltimateLyricsProvider::RuleItem::Range,
+ attr.value("begin").toString(), attr.value("end").toString())
+ : UltimateLyricsProvider::RuleItem(UltimateLyricsProvider::RuleItem::XmlTag,
+ attr.value("begin").toString()));
+ }
+ else if (attr.hasAttribute("container")) {
+ ret << UltimateLyricsProvider::RuleItem(UltimateLyricsProvider::RuleItem::Container, attr.value("container").toString());
+ }
+ else if (attr.hasAttribute("unescape")) {
+ ret << UltimateLyricsProvider::RuleItem(UltimateLyricsProvider::RuleItem::Unescape, attr.value("unescape").toString());
}
}
reader->skipCurrentElement();
@@ -81,6 +93,8 @@ static UltimateLyricsProvider* parseProvider(QXmlStreamReader* reader)
scraper->setName(attributes.value("name").toString());
scraper->setCharset(attributes.value("charset").toString());
scraper->setUrl(attributes.value("url").toString());
+ scraper->setPageUrl(attributes.value("pageUrl").toString());
+ scraper->setProvidesMarkup(QLatin1String("true") == attributes.value("markup"));
while (!reader->atEnd()) {
reader->readNext();
@@ -184,7 +198,7 @@ void UltimateLyrics::load()
UltimateLyricsProvider* provider = parseProvider(&reader);
if (provider) {
providers << provider;
- connect(provider, SIGNAL(lyricsReady(int, QString)), this, SIGNAL(lyricsReady(int, QString)));
+ connect(provider, SIGNAL(lyricsReady(int, QString, QUrl)), this, SIGNAL(lyricsReady(int, QString, QUrl)));
providerNames.insert(name);
}
}
diff --git a/context/ultimatelyrics.h b/context/ultimatelyrics.h
index 3c175105e..31607ada6 100644
--- a/context/ultimatelyrics.h
+++ b/context/ultimatelyrics.h
@@ -25,6 +25,7 @@
#define ULTIMATELYRICS_H
#include
+#include
class UltimateLyricsProvider;
@@ -41,7 +42,7 @@ class UltimateLyrics : public QObject {
void setEnabled(const QStringList& enabled);
Q_SIGNALS:
- void lyricsReady(int id, const QString& data);
+ void lyricsReady(int id, const QString& data, const QUrl& source);
private:
UltimateLyricsProvider* providerByName(const QString& name) const;
diff --git a/context/ultimatelyricsprovider.cpp b/context/ultimatelyricsprovider.cpp
index 91b832ea9..e7ecc4d4a 100644
--- a/context/ultimatelyricsprovider.cpp
+++ b/context/ultimatelyricsprovider.cpp
@@ -121,7 +121,7 @@ static QString extract(const QString& source, const QString& begin, const QStrin
}
DBUG << "Found match";
- return source.mid(beginIdx, endIdx - beginIdx - 1);
+ return source.mid(beginIdx, endIdx - beginIdx);
}
static QRegularExpression xmlTagRegex = QRegularExpression("<(\\w+).*>");
@@ -138,6 +138,156 @@ static QString extractXmlTag(const QString& source, const QString& tag)
return extract(source, tag, "", true);
}
+// Decode a JSON string literal. Sites that render client-side (e.g. Musixmatch) embed the
+// lyrics in a JSON blob rather than in markup, so what the extract rules pull out still has
+// its escapes intact.
+static QString jsonUnescape(const QString& source)
+{
+ QString ret;
+ ret.reserve(source.length());
+
+ for (int i = 0; i < source.length(); ++i) {
+ if (QLatin1Char('\\') != source.at(i) || i + 1 >= source.length()) {
+ ret += source.at(i);
+ continue;
+ }
+
+ QChar esc = source.at(++i);
+ switch (esc.toLatin1()) {
+ case 'n': ret += QLatin1Char('\n'); break;
+ case 't': ret += QLatin1Char('\t'); break;
+ case 'r': ret += QLatin1Char('\r'); break;
+ case 'b': ret += QLatin1Char('\b'); break;
+ case 'f': ret += QLatin1Char('\f'); break;
+ case 'u': {
+ bool ok = false;
+ ushort code = i + 4 < source.length() ? source.mid(i + 1, 4).toUShort(&ok, 16) : 0;
+ if (ok) {
+ ret += QChar(code);
+ i += 4;
+ }
+ else {
+ ret += esc;
+ }
+ break;
+ }
+ // Covers \" \\ \/ - and leaves anything unrecognised as-is.
+ default: ret += esc; break;
+ }
+ }
+ return ret;
+}
+
+// Matches the opening tag of an element carrying the named attribute. The attribute must be a
+// whitespace-separated name followed by '=', so that a name is not matched by a longer one that
+// merely starts with it - a word boundary would not do, as '-' ends a word.
+static QRegularExpression containerRegex(const QString& attribute)
+{
+ return QRegularExpression("<(\\w+)\\b[^>]*\\s" + QRegularExpression::escape(attribute) + "\\s*=[^>]*>",
+ QRegularExpression::CaseInsensitiveOption);
+}
+
+// Given the opening tag matched by containerRegex(), walks the same tag in and out until the
+// element's own closing tag is reached, so that nested markup does not terminate it early.
+// Sets contentEnd to the start of that closing tag, and elementEnd to just past it.
+static bool findContainerEnd(const QString& source, const QRegularExpressionMatch& open, int& contentEnd, int& elementEnd)
+{
+ QRegularExpression nestRegex("<(/?)" + open.captured(1) + "\\b[^>]*>", QRegularExpression::CaseInsensitiveOption);
+ int scan = open.capturedEnd();
+ int depth = 1;
+
+ while (depth > 0) {
+ auto nest = nestRegex.match(source, scan);
+ if (!nest.hasMatch()) {
+ DBUG << "Failed to find end of container";
+ return false;
+ }
+ if (nest.captured(1).isEmpty()) {
+ ++depth;
+ }
+ else if (0 == --depth) {
+ contentEnd = nest.capturedStart();
+ elementEnd = nest.capturedEnd();
+ }
+ scan = nest.capturedEnd();
+ }
+ return true;
+}
+
+// Extract the contents of every element carrying the named attribute. Used for sites (e.g.
+// Genius) that split the lyrics over several containers whose class names are build-specific,
+// and whose contents hold nested markup - so neither a literal tag match nor a naive begin/end
+// pair works.
+static QString extractContainers(const QString& source, const QString& attribute)
+{
+ DBUG << "Looking for containers with" << attribute;
+ QRegularExpression openRegex = containerRegex(attribute);
+ QStringList parts;
+ int pos = 0;
+
+ while (pos < source.length()) {
+ auto open = openRegex.match(source, pos);
+ if (!open.hasMatch()) {
+ break;
+ }
+
+ int contentEnd = -1;
+ int elementEnd = -1;
+ if (!findContainerEnd(source, open, contentEnd, elementEnd)) {
+ break;
+ }
+
+ // Empty containers are used as placeholders around ad slots - skip them, so that
+ // joining does not introduce stray breaks between the sections that do have lyrics.
+ QString content = source.mid(open.capturedEnd(), contentEnd - open.capturedEnd());
+ if (!content.trimmed().isEmpty()) {
+ parts << content;
+ }
+ pos = elementEnd;
+ }
+
+ if (parts.isEmpty()) {
+ DBUG << "Failed to find container";
+ return QString();
+ }
+
+ DBUG << "Found" << parts.length() << "container(s)";
+ return parts.join(QLatin1String("
"));
+}
+
+// Remove every element carrying the named attribute, contents included. Genius marks the
+// page furniture it embeds within the lyrics containers (contributor count, song title,
+// summary) with such an attribute, and it has to go along with the markup it sits in.
+static QString excludeContainers(const QString& source, const QString& attribute)
+{
+ DBUG << "Looking for containers with" << attribute;
+ QRegularExpression openRegex = containerRegex(attribute);
+ QString ret = source;
+ int removed = 0;
+ int pos = 0;
+
+ while (pos < ret.length()) {
+ auto open = openRegex.match(ret, pos);
+ if (!open.hasMatch()) {
+ break;
+ }
+
+ int contentEnd = -1;
+ int elementEnd = -1;
+ if (!findContainerEnd(ret, open, contentEnd, elementEnd)) {
+ break;
+ }
+
+ // Any nested match is removed along with this one, so resume from where it started.
+ ret.remove(open.capturedStart(), elementEnd - open.capturedStart());
+ pos = open.capturedStart();
+ ++removed;
+ }
+
+ DBUG << "Removed" << removed << "container(s)";
+ return ret;
+}
+
static QString exclude(const QString& source, const QString& begin, const QString& end)
{
int beginIdx = source.indexOf(begin, 0, Qt::CaseInsensitive);
@@ -166,11 +316,21 @@ static QString excludeXmlTag(const QString& source, const QString& tag)
static void applyExtractRule(const UltimateLyricsProvider::Rule& rule, QString& content, const Song& song)
{
for (const UltimateLyricsProvider::RuleItem& item : rule) {
- if (item.second.isNull()) {
- content = extractXmlTag(content, doTagReplace(item.first, song));
- }
- else {
- content = extract(content, doTagReplace(item.first, song), doTagReplace(item.second, song));
+ switch (item.type) {
+ case UltimateLyricsProvider::RuleItem::XmlTag:
+ content = extractXmlTag(content, doTagReplace(item.begin, song));
+ break;
+ case UltimateLyricsProvider::RuleItem::Range:
+ content = extract(content, doTagReplace(item.begin, song), doTagReplace(item.end, song));
+ break;
+ case UltimateLyricsProvider::RuleItem::Container:
+ content = extractContainers(content, doTagReplace(item.begin, song));
+ break;
+ case UltimateLyricsProvider::RuleItem::Unescape:
+ if (QLatin1String("json") == item.begin) {
+ content = jsonUnescape(content);
+ }
+ break;
}
}
}
@@ -178,11 +338,19 @@ static void applyExtractRule(const UltimateLyricsProvider::Rule& rule, QString&
static void applyExcludeRule(const UltimateLyricsProvider::Rule& rule, QString& content, const Song& song)
{
for (const UltimateLyricsProvider::RuleItem& item : rule) {
- if (item.second.isNull()) {
- content = excludeXmlTag(content, doTagReplace(item.first, song));
- }
- else {
- content = exclude(content, doTagReplace(item.first, song), doTagReplace(item.second, song));
+ switch (item.type) {
+ case UltimateLyricsProvider::RuleItem::XmlTag:
+ content = excludeXmlTag(content, doTagReplace(item.begin, song));
+ break;
+ case UltimateLyricsProvider::RuleItem::Range:
+ content = exclude(content, doTagReplace(item.begin, song), doTagReplace(item.end, song));
+ break;
+ case UltimateLyricsProvider::RuleItem::Container:
+ content = excludeContainers(content, doTagReplace(item.begin, song));
+ break;
+ case UltimateLyricsProvider::RuleItem::Unescape:
+ // Not meaningful as an exclusion - ignored.
+ break;
}
}
}
@@ -201,7 +369,7 @@ static bool tryWithoutThe(const Song& s)
}
UltimateLyricsProvider::UltimateLyricsProvider()
- : enabled(true), relevance(0)
+ : enabled(true), relevance(0), markup(false)
{
}
@@ -218,72 +386,69 @@ QString UltimateLyricsProvider::displayName() const
return n;
}
-void UltimateLyricsProvider::fetchInfo(int id, Song metadata, bool removeThe)
+QUrl UltimateLyricsProvider::buildUrl(const QString& templateUrl, const QString& artist, const QString& title, const Song& metadata) const
{
- auto converter = QStringDecoder(charset.toLatin1().constData(), QStringConverter::Flag::Default);
-
- if (!converter.isValid()) {
- emit lyricsReady(id, QString());
- return;
- }
-
- QString artistFixed = metadata.basicArtist();
- QString titleFixed = metadata.basicTitle();
- QString urlText(url);
-
- if (removeThe && artistFixed.startsWith(constThe)) {
- artistFixed = artistFixed.mid(constThe.length());
- }
-
- if (QLatin1String("lyrics.wikia.com") == name) {
- QUrl url(urlText);
- QUrlQuery query;
-
- query.addQueryItem(QLatin1String("artist"), artistFixed);
- query.addQueryItem(QLatin1String("song"), titleFixed);
- query.addQueryItem(QLatin1String("func"), QLatin1String("getSong"));
- query.addQueryItem(QLatin1String("fmt"), QLatin1String("xml"));
- url.setQuery(query);
-
- NetworkJob* reply = NetworkAccessManager::self()->get(url);
- requests[reply] = id;
- connect(reply, SIGNAL(finished()), this, SLOT(wikiMediaSearchResponse()));
- return;
- }
-
- metadata.priority = removeThe ? 1 : 0;// HACK Use this to indicate if searching without 'The '
- songs.insert(id, metadata);
+ QString urlText(templateUrl);
// Fill in fields in the URL
bool urlContainsDetails = urlText.contains(QLatin1Char('{'));
if (urlContainsDetails) {
- doUrlReplace(constArtistArg, artistFixed, urlText);
- doUrlReplace(constArtistLowerArg, artistFixed.toLower(), urlText);
- doUrlReplace(constArtistLowerNoSpaceArg, noSpace(artistFixed.toLower()), urlText);
- doUrlReplace(constArtistFirstCharArg, firstChar(artistFixed), urlText);
+ doUrlReplace(constArtistArg, artist, urlText);
+ doUrlReplace(constArtistLowerArg, artist.toLower(), urlText);
+ doUrlReplace(constArtistLowerNoSpaceArg, noSpace(artist.toLower()), urlText);
+ doUrlReplace(constArtistFirstCharArg, firstChar(artist), urlText);
doUrlReplace(constAlbumArg, metadata.album, urlText);
doUrlReplace(constAlbumLowerArg, metadata.album.toLower(), urlText);
doUrlReplace(constAlbumLowerNoSpaceArg, noSpace(metadata.album.toLower()), urlText);
- doUrlReplace(constTitleArg, titleFixed, urlText);
- doUrlReplace(constTitleLowerArg, titleFixed.toLower(), urlText);
- doUrlReplace(constTitleCaseArg, titleCase(titleFixed), urlText);
+ doUrlReplace(constTitleArg, title, urlText);
+ doUrlReplace(constTitleLowerArg, title.toLower(), urlText);
+ doUrlReplace(constTitleCaseArg, titleCase(title), urlText);
doUrlReplace(constYearArg, QString::number(metadata.year), urlText);
doUrlReplace(constTrackNoArg, QString::number(metadata.track), urlText);
}
// For some reason Qt messes up the ? -> %3F and & -> %26 conversions - by placing 25 after the %
// So, try and revert this...
- QUrl url(urlText);
+ QUrl built(urlText);
if (urlContainsDetails) {
- QByteArray data = url.toEncoded();
+ QByteArray data = built.toEncoded();
data.replace("%253F", "%3F");
data.replace("%253f", "%3f");
data.replace("%2526", "%26");
- url = QUrl::fromEncoded(data, QUrl::StrictMode);
+ built = QUrl::fromEncoded(data, QUrl::StrictMode);
}
- QNetworkRequest req(url);
+ return built;
+}
+
+void UltimateLyricsProvider::fetchInfo(int id, Song metadata, bool removeThe)
+{
+ auto converter = QStringDecoder(charset.toLatin1().constData(), QStringConverter::Flag::Default);
+
+ if (!converter.isValid()) {
+ emit lyricsReady(id, QString(), QUrl());
+ return;
+ }
+
+ QString artistFixed = metadata.basicArtist();
+ QString titleFixed = metadata.basicTitle();
+
+ if (removeThe && artistFixed.startsWith(constThe)) {
+ artistFixed = artistFixed.mid(constThe.length());
+ }
+
+ metadata.priority = removeThe ? 1 : 0;// HACK Use this to indicate if searching without 'The '
+ songs.insert(id, metadata);
+
+ QUrl fetchUrl = buildUrl(url, artistFixed, titleFixed, metadata);
+
+ // Remember where these lyrics came from, so that we can offer a link to the source. For
+ // providers queried through an API the fetched url is of no use to the user, so those supply a
+ // separate, human readable, page url instead.
+ sourceUrls.insert(id, pageUrl.isEmpty() ? fetchUrl : buildUrl(pageUrl, artistFixed, titleFixed, metadata));
+
+ QNetworkRequest req(fetchUrl);
req.setRawHeader("User-Agent", "Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0");
NetworkJob* reply = NetworkAccessManager::self()->get(req);
requests[reply] = id;
@@ -300,6 +465,7 @@ void UltimateLyricsProvider::abort()
}
requests.clear();
songs.clear();
+ sourceUrls.clear();
}
void UltimateLyricsProvider::wikiMediaSearchResponse()
@@ -314,11 +480,12 @@ void UltimateLyricsProvider::wikiMediaSearchResponse()
if (!reply->ok()) {
Song song = songs.take(id);
+ sourceUrls.remove(id);
if (tryWithoutThe(song)) {
fetchInfo(id, song, true);
}
else {
- emit lyricsReady(id, QString());
+ emit lyricsReady(id, QString(), QUrl());
}
return;
}
@@ -345,7 +512,8 @@ void UltimateLyricsProvider::wikiMediaSearchResponse()
connect(reply, SIGNAL(finished()), this, SLOT(wikiMediaLyricsFetched()));
}
else {
- emit lyricsReady(id, QString());
+ sourceUrls.remove(id);
+ emit lyricsReady(id, QString(), QUrl());
}
}
@@ -361,11 +529,12 @@ void UltimateLyricsProvider::wikiMediaLyricsFetched()
if (!reply->ok()) {
Song song = songs.take(id);
+ sourceUrls.remove(id);
if (tryWithoutThe(song)) {
fetchInfo(id, song, true);
}
else {
- emit lyricsReady(id, QString());
+ emit lyricsReady(id, QString(), QUrl());
}
return;
}
@@ -374,7 +543,7 @@ void UltimateLyricsProvider::wikiMediaLyricsFetched()
QString contents = fromCharset(reply->readAll());
contents = contents.replace("
", "
");
DBUG << name << "response" << contents;
- emit lyricsReady(id, extract(contents, QLatin1String("<lyrics>"), QLatin1String("</lyrics>")));
+ emit lyricsReady(id, extract(contents, QLatin1String("<lyrics>"), QLatin1String("</lyrics>")), sourceUrls.take(id));
}
void UltimateLyricsProvider::lyricsFetched()
@@ -387,6 +556,7 @@ void UltimateLyricsProvider::lyricsFetched()
int id = requests.take(reply);
reply->deleteLater();
Song song = songs.take(id);
+ QUrl source = sourceUrls.take(id);
if (!reply->ok()) {
//emit Finished(id);
@@ -394,7 +564,7 @@ void UltimateLyricsProvider::lyricsFetched()
fetchInfo(id, song, true);
}
else {
- emit lyricsReady(id, QString());
+ emit lyricsReady(id, QString(), QUrl());
}
return;
}
@@ -413,7 +583,7 @@ void UltimateLyricsProvider::lyricsFetched()
fetchInfo(id, song, true);
}
else {
- emit lyricsReady(id, QString());
+ emit lyricsReady(id, QString(), QUrl());
}
return;
}
@@ -449,7 +619,7 @@ void UltimateLyricsProvider::lyricsFetched()
fetchInfo(id, song, true);
}
else {
- emit lyricsReady(id, lyrics);
+ emit lyricsReady(id, lyrics, lyrics.isEmpty() ? QUrl() : source);
}
}
diff --git a/context/ultimatelyricsprovider.h b/context/ultimatelyricsprovider.h
index 80378dbb9..f56410f7d 100644
--- a/context/ultimatelyricsprovider.h
+++ b/context/ultimatelyricsprovider.h
@@ -30,6 +30,7 @@
#include
#include
#include
+#include
class NetworkJob;
@@ -42,12 +43,34 @@ class UltimateLyricsProvider : public QObject {
UltimateLyricsProvider();
~UltimateLyricsProvider() override;
- typedef QPair RuleItem;
+ struct RuleItem {
+ enum Type {
+ XmlTag, //