From b1965e92a2e859c132397ddc70877c795e460a99 Mon Sep 17 00:00:00 2001 From: Abdallah Ahmed Date: Tue, 10 Aug 2021 12:49:23 +0200 Subject: [PATCH 01/11] refactor(Bibliography Renderer): Abstract collecting references into a separate function --- .../extract-references.ts | 42 +++++++++++++++++++ .../render-list-content-script.ts | 35 ++-------------- 2 files changed, 45 insertions(+), 32 deletions(-) create mode 100644 src/ui/bibliography-renderer/extract-references.ts diff --git a/src/ui/bibliography-renderer/extract-references.ts b/src/ui/bibliography-renderer/extract-references.ts new file mode 100644 index 0000000..edb6e9f --- /dev/null +++ b/src/ui/bibliography-renderer/extract-references.ts @@ -0,0 +1,42 @@ +import { Reference } from "../../model/reference.model"; + +/** + * Given a list of markdown tokens and their children, + * returns a list of reference IDs that exists in the markdown tree + * Uses Depth-First-Search + */ +export function extractReferences(tokens: any[]): Reference[] { + const ids: Reference[] = []; + DFS(tokens); + + function DFS(children: any[]): void { + if (!children) return; + + /* Search for three consecutive tokens: "link_open", "text", and "link_close" */ + for (let i = 1; i < children.length - 1; i++) { + const curr = children[i], + prev = children[i - 1], + next = children[i + 1]; + if ( + prev["type"] === "link_open" && + curr["type"] === "text" && + next["type"] === "link_close" && + curr.content && + curr.content.length > 1 && + curr.content.startsWith("@") + ) { + const id = curr.content.substring(1); + ids.push(id); + } else { + if (curr["children"]) DFS(curr["children"]); + } + } + // first and last child that were not traversed previously + const last = children[children.length - 1], + first = children[0]; + if (last["children"]) DFS(last["children"]); + if (first["children"]) DFS(first["children"]); + } + + return ids; +} diff --git a/src/ui/bibliography-renderer/render-list-content-script.ts b/src/ui/bibliography-renderer/render-list-content-script.ts index e9ffadb..dad40f6 100644 --- a/src/ui/bibliography-renderer/render-list-content-script.ts +++ b/src/ui/bibliography-renderer/render-list-content-script.ts @@ -1,4 +1,5 @@ import { Reference } from "../../model/reference.model"; +import { extractReferences } from "./extract-references"; export default function (context) { return { @@ -7,38 +8,8 @@ export default function (context) { /* Appends a new custom token for references list */ markdownIt.core.ruler.push("reference_list", async (state) => { - /* Collect references from the note body using Depth-first-search */ - const ids: Reference[] = []; - dfs(state.tokens); - - function dfs(children: any[]): void { - if (!children) return; - - /* Search for three consecutive tokens: "link_open", "text", and "link_close" */ - for (let i = 1; i < children.length - 1; i++) { - const curr = children[i], - prev = children[i - 1], - next = children[i + 1]; - if ( - prev["type"] === "link_open" && - curr["type"] === "text" && - next["type"] === "link_close" && - curr.content && - curr.content.length > 1 && - curr.content.startsWith("@") - ) { - const id = curr.content.substring(1); - ids.push(id); - } else { - if (curr["children"]) dfs(curr["children"]); - } - } - // first and last child that were not traversed previously - const last = children[children.length - 1], - first = children[0]; - if (last["children"]) dfs(last["children"]); - if (first["children"]) dfs(first["children"]); - } + /* Collect references from the note body */ + const ids: Reference[] = extractReferences(state.tokens); /* Append reference_list token */ let token = new state.Token("reference_list", "", 0); From 85cdd8fc48269e0dce9844d59fcb12100132fabd Mon Sep 17 00:00:00 2001 From: Abdallah Ahmed Date: Tue, 10 Aug 2021 17:27:31 +0200 Subject: [PATCH 02/11] feat(Bibliography Renderer): Search for words that starts with @ --- .../extract-references.ts | 48 ++++++++----------- .../render-list-content-script.ts | 4 +- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/src/ui/bibliography-renderer/extract-references.ts b/src/ui/bibliography-renderer/extract-references.ts index edb6e9f..a0eff83 100644 --- a/src/ui/bibliography-renderer/extract-references.ts +++ b/src/ui/bibliography-renderer/extract-references.ts @@ -1,41 +1,33 @@ -import { Reference } from "../../model/reference.model"; - /** * Given a list of markdown tokens and their children, * returns a list of reference IDs that exists in the markdown tree * Uses Depth-First-Search */ -export function extractReferences(tokens: any[]): Reference[] { - const ids: Reference[] = []; +export function extractReferences(tokens: any[]): string[] { + const ids: string[] = []; DFS(tokens); - function DFS(children: any[]): void { - if (!children) return; - - /* Search for three consecutive tokens: "link_open", "text", and "link_close" */ - for (let i = 1; i < children.length - 1; i++) { - const curr = children[i], - prev = children[i - 1], - next = children[i + 1]; + /* Collect all words that starts with @ */ + function DFS(nodes: any[]): void { + nodes.forEach((node) => { if ( - prev["type"] === "link_open" && - curr["type"] === "text" && - next["type"] === "link_close" && - curr.content && - curr.content.length > 1 && - curr.content.startsWith("@") + node["type"] === "text" && + node.content && + node.content.length > 1 ) { - const id = curr.content.substring(1); - ids.push(id); - } else { - if (curr["children"]) DFS(curr["children"]); + /* A text token might contain several words separable by a space */ + const content: string = node.content; + content + .split(" ") + .filter((word) => word.startsWith("@")) + .map((word) => word.substring(1)) // Remove the @ + .forEach((word) => ids.push(word)); + } + + if (node["children"]) { + DFS(node["children"]); } - } - // first and last child that were not traversed previously - const last = children[children.length - 1], - first = children[0]; - if (last["children"]) DFS(last["children"]); - if (first["children"]) DFS(first["children"]); + }); } return ids; diff --git a/src/ui/bibliography-renderer/render-list-content-script.ts b/src/ui/bibliography-renderer/render-list-content-script.ts index dad40f6..b444276 100644 --- a/src/ui/bibliography-renderer/render-list-content-script.ts +++ b/src/ui/bibliography-renderer/render-list-content-script.ts @@ -7,9 +7,9 @@ export default function (context) { const contentScriptId = context.contentScriptId; /* Appends a new custom token for references list */ - markdownIt.core.ruler.push("reference_list", async (state) => { + markdownIt.core.ruler.push("reference_list", (state) => { /* Collect references from the note body */ - const ids: Reference[] = extractReferences(state.tokens); + const ids: string[] = extractReferences(state.tokens); /* Append reference_list token */ let token = new state.Token("reference_list", "", 0); From dbab3be7b20d68082cd403d2e5c29bc4a5472ee0 Mon Sep 17 00:00:00 2001 From: Abdallah Ahmed Date: Wed, 11 Aug 2021 02:47:49 +0200 Subject: [PATCH 03/11] feat(Bibliography Renderer): Extract all cite keys from the note body --- .../extract-references.ts | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/ui/bibliography-renderer/extract-references.ts b/src/ui/bibliography-renderer/extract-references.ts index a0eff83..bf1ec89 100644 --- a/src/ui/bibliography-renderer/extract-references.ts +++ b/src/ui/bibliography-renderer/extract-references.ts @@ -1,34 +1,35 @@ /** * Given a list of markdown tokens and their children, * returns a list of reference IDs that exists in the markdown tree - * Uses Depth-First-Search + * Uses recursive Depth-First-Search */ export function extractReferences(tokens: any[]): string[] { const ids: string[] = []; - DFS(tokens); + DFS(tokens, ids); + return ids; +} - /* Collect all words that starts with @ */ - function DFS(nodes: any[]): void { - nodes.forEach((node) => { - if ( - node["type"] === "text" && - node.content && - node.content.length > 1 - ) { - /* A text token might contain several words separable by a space */ - const content: string = node.content; - content - .split(" ") - .filter((word) => word.startsWith("@")) +function DFS(nodes: any[], ids): void { + /* Collect all words that matches the below regular expression */ + const referencePattern = /@(\w|:|\?|\-)+/g; + nodes.forEach((node) => { + if ( + node["type"] === "text" && + node.content && + node.content.length > 1 + ) { + /* A text token might contain several words separable by a space */ + const content: string = node.content; + const matches = content.match(referencePattern); + if (matches && matches.length > 0) { + matches .map((word) => word.substring(1)) // Remove the @ .forEach((word) => ids.push(word)); } + } - if (node["children"]) { - DFS(node["children"]); - } - }); - } - - return ids; + if (node["children"]) { + DFS(node["children"], ids); + } + }); } From 45138b4206af6ffdd491406d0083d287a2c03b29 Mon Sep 17 00:00:00 2001 From: Abdallah Ahmed Date: Wed, 11 Aug 2021 02:57:00 +0200 Subject: [PATCH 04/11] perf(Bibliography Renderer): Slight performance improvements --- .../extract-references.ts | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/src/ui/bibliography-renderer/extract-references.ts b/src/ui/bibliography-renderer/extract-references.ts index bf1ec89..9d2f924 100644 --- a/src/ui/bibliography-renderer/extract-references.ts +++ b/src/ui/bibliography-renderer/extract-references.ts @@ -5,31 +5,30 @@ */ export function extractReferences(tokens: any[]): string[] { const ids: string[] = []; - DFS(tokens, ids); - return ids; -} - -function DFS(nodes: any[], ids): void { - /* Collect all words that matches the below regular expression */ const referencePattern = /@(\w|:|\?|\-)+/g; - nodes.forEach((node) => { - if ( - node["type"] === "text" && - node.content && - node.content.length > 1 - ) { - /* A text token might contain several words separable by a space */ - const content: string = node.content; - const matches = content.match(referencePattern); - if (matches && matches.length > 0) { - matches - .map((word) => word.substring(1)) // Remove the @ - .forEach((word) => ids.push(word)); + + /* Collect all words that matches the regular expression */ + DFS(tokens); + + function DFS(nodes: any[]): void { + nodes.forEach((node) => { + if ( + node["type"] === "text" && + node.content && + node.content.length > 1 + ) { + /* If matches found, add them to IDs without @ */ + const matches = node.content.match(referencePattern); + if (matches && matches.length > 0) { + matches.forEach((word) => ids.push(word.substring(1))); + } + } + + if (node["children"]) { + DFS(node["children"]); } - } + }); + } - if (node["children"]) { - DFS(node["children"], ids); - } - }); + return ids; } From c2a1de7e7b33ee0e148a0a6f639b5f15eb5fa76e Mon Sep 17 00:00:00 2001 From: Abdallah Ahmed Date: Wed, 11 Aug 2021 15:57:30 +0200 Subject: [PATCH 05/11] fix(Bibliography Renderer): Reference not found When the user types a word that starts with "@", the content script reads this as a potential cite key and tries to get the actual reference directly. Of course, the word does not have to be a valid cite key, which causes the data store to throw "Reference not Found" Exception --- src/ui/bibliography-renderer/index.ts | 17 ++++++++++++++--- .../render-list-content-script.ts | 1 - 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/ui/bibliography-renderer/index.ts b/src/ui/bibliography-renderer/index.ts index 619d0c6..5216f46 100644 --- a/src/ui/bibliography-renderer/index.ts +++ b/src/ui/bibliography-renderer/index.ts @@ -1,10 +1,10 @@ import joplin from "api"; import { ContentScriptType } from "api/types"; +import { DataStore } from "../../data/data-store"; import { CSLProcessor } from "../../util/csl-processor"; import { REFERENCE_LIST_CONTENT_SCRIPT_ID, SETTINGS_CSL_FILE_PATH_ID, - MESSAGE_RESTART_APP, } from "../../constants"; /** @@ -26,11 +26,22 @@ export async function registerBibliographyRenderer(): Promise { REFERENCE_LIST_CONTENT_SCRIPT_ID, (IDs: string[]) => { - IDs = [...new Set(IDs)]; // Filter duplicate references + /** + * Filter duplicate IDs + * Filter fake IDs (IDs that don't correspond to actual reference objects) + */ + IDs = [...new Set(IDs)].filter((id) => { + try { + DataStore.getReferenceById(id); + return true; + } catch (e) { + return false; + } + }); /** * Apply the specified citation style to the references - * Note: Does html-encoding by default + * Does html-encoding by default */ return processor.formatRefs(IDs); } diff --git a/src/ui/bibliography-renderer/render-list-content-script.ts b/src/ui/bibliography-renderer/render-list-content-script.ts index b444276..5e0d6fb 100644 --- a/src/ui/bibliography-renderer/render-list-content-script.ts +++ b/src/ui/bibliography-renderer/render-list-content-script.ts @@ -1,4 +1,3 @@ -import { Reference } from "../../model/reference.model"; import { extractReferences } from "./extract-references"; export default function (context) { From 35ff70bb85976f64e727d33a0ad9b54fcfd6cfc9 Mon Sep 17 00:00:00 2001 From: Abdallah Ahmed Date: Fri, 13 Aug 2021 02:06:07 +0200 Subject: [PATCH 06/11] refactor(Bibliography Renderer): Distinguish between message types --- src/ui/bibliography-renderer/Message.ts | 2 + src/ui/bibliography-renderer/index.ts | 50 +++++++++++-------- .../render-list-content-script.ts | 18 ++++--- 3 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 src/ui/bibliography-renderer/Message.ts diff --git a/src/ui/bibliography-renderer/Message.ts b/src/ui/bibliography-renderer/Message.ts new file mode 100644 index 0000000..b18f5b0 --- /dev/null +++ b/src/ui/bibliography-renderer/Message.ts @@ -0,0 +1,2 @@ +export const FORMAT_REFERENCES: string = "format_references"; +export const GET_REFERENCES_BY_ID: string = "get_reference_by_id"; diff --git a/src/ui/bibliography-renderer/index.ts b/src/ui/bibliography-renderer/index.ts index 5216f46..8f89370 100644 --- a/src/ui/bibliography-renderer/index.ts +++ b/src/ui/bibliography-renderer/index.ts @@ -6,6 +6,7 @@ import { REFERENCE_LIST_CONTENT_SCRIPT_ID, SETTINGS_CSL_FILE_PATH_ID, } from "../../constants"; +import { FORMAT_REFERENCES, GET_REFERENCES_BY_ID } from "./Message"; /** * Render the full list of references at the end of the note viewer @@ -18,32 +19,39 @@ export async function registerBibliographyRenderer(): Promise { "./ui/bibliography-renderer/render-list-content-script.js" ); - /** - * Format the references according to the style specified by the user - */ const processor = CSLProcessor.getInstance(); + + /* Handle messages sent by the content script */ await joplin.contentScripts.onMessage( REFERENCE_LIST_CONTENT_SCRIPT_ID, - (IDs: string[]) => { - /** - * Filter duplicate IDs - * Filter fake IDs (IDs that don't correspond to actual reference objects) - */ - IDs = [...new Set(IDs)].filter((id) => { - try { - DataStore.getReferenceById(id); - return true; - } catch (e) { - return false; - } - }); + (req: { type: string }) => { + switch (req.type) { + /** + * Format the references according to the style specified by the user + */ + case FORMAT_REFERENCES: + /** + * Filter duplicate IDs + * Filter fake IDs (IDs that don't correspond to actual reference objects) + */ + let IDs: string[] = req["IDs"]; + IDs = [...new Set(IDs)].filter((id) => { + try { + DataStore.getReferenceById(id); + return true; + } catch (e) { + return false; + } + }); - /** - * Apply the specified citation style to the references - * Does html-encoding by default - */ - return processor.formatRefs(IDs); + /** + * Apply the specified citation style to the references + * Does html-encoding by default + */ + return processor.formatRefs(IDs); + break; + } } ); setProcessorStyle(processor); diff --git a/src/ui/bibliography-renderer/render-list-content-script.ts b/src/ui/bibliography-renderer/render-list-content-script.ts index 5e0d6fb..d3b9a08 100644 --- a/src/ui/bibliography-renderer/render-list-content-script.ts +++ b/src/ui/bibliography-renderer/render-list-content-script.ts @@ -1,4 +1,5 @@ import { extractReferences } from "./extract-references"; +import { FORMAT_REFERENCES, GET_REFERENCES_BY_ID } from "./Message"; export default function (context) { return { @@ -16,17 +17,20 @@ export default function (context) { state.tokens.push(token); }); - /* Define how to render the previously defined token */ - markdownIt.renderer.rules["reference_list"] = renderReferenceList; - - function renderReferenceList(tokens, idx, options) { + /* Define how to render the reference_list token */ + markdownIt.renderer.rules["reference_list"] = function ( + tokens, + idx, + options + ) { let IDs: string[] = tokens[idx]["attrs"][0][1]; if (IDs.length === 0) return ""; const script: string = ` - webviewApi.postMessage("${contentScriptId}", ${JSON.stringify( - IDs - )}).then(html => { + webviewApi.postMessage("${contentScriptId}", ${JSON.stringify({ + type: FORMAT_REFERENCES, + IDs, + })}).then(html => { const referenceListView = document.getElementById("references_list"); const referenceTitleView = document.getElementById("references_title"); From 9e353e64adf3e11eac94d60818562d7f372bb5a3 Mon Sep 17 00:00:00 2001 From: Abdallah Ahmed Date: Sat, 14 Aug 2021 16:36:41 +0200 Subject: [PATCH 07/11] feat(Bibliography Renderer): Replace inline references with a fixed text --- src/ui/bibliography-renderer/Message.ts | 2 +- src/ui/bibliography-renderer/index.ts | 12 ++- .../render-inline-references.ts | 77 +++++++++++++++++++ .../render-list-content-script.ts | 14 +++- 4 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 src/ui/bibliography-renderer/render-inline-references.ts diff --git a/src/ui/bibliography-renderer/Message.ts b/src/ui/bibliography-renderer/Message.ts index b18f5b0..8f52ba8 100644 --- a/src/ui/bibliography-renderer/Message.ts +++ b/src/ui/bibliography-renderer/Message.ts @@ -1,2 +1,2 @@ export const FORMAT_REFERENCES: string = "format_references"; -export const GET_REFERENCES_BY_ID: string = "get_reference_by_id"; +export const GET_REFERENCE_BY_ID: string = "get_reference_by_id"; diff --git a/src/ui/bibliography-renderer/index.ts b/src/ui/bibliography-renderer/index.ts index 8f89370..5b14bde 100644 --- a/src/ui/bibliography-renderer/index.ts +++ b/src/ui/bibliography-renderer/index.ts @@ -6,7 +6,7 @@ import { REFERENCE_LIST_CONTENT_SCRIPT_ID, SETTINGS_CSL_FILE_PATH_ID, } from "../../constants"; -import { FORMAT_REFERENCES, GET_REFERENCES_BY_ID } from "./Message"; +import { FORMAT_REFERENCES, GET_REFERENCE_BY_ID } from "./Message"; /** * Render the full list of references at the end of the note viewer @@ -51,6 +51,16 @@ export async function registerBibliographyRenderer(): Promise { */ return processor.formatRefs(IDs); break; + + case GET_REFERENCE_BY_ID: + console.log(req); + const id = req["id"]; + try { + return DataStore.getReferenceById(id); + } catch (e) { + return null; + } + break; } } ); diff --git a/src/ui/bibliography-renderer/render-inline-references.ts b/src/ui/bibliography-renderer/render-inline-references.ts new file mode 100644 index 0000000..552429e --- /dev/null +++ b/src/ui/bibliography-renderer/render-inline-references.ts @@ -0,0 +1,77 @@ +import { GET_REFERENCE_BY_ID } from "./Message"; + +/** + * Given a list of markdown tokens and their children, + * formats all inline references according to the syntax given here: + * https://discourse.joplinapp.org/t/customize-the-rendering-of-inline-references-spec/19294?u=xuser5000 + * Uses recursive Depth-First-Search + */ +export function renderInlineReferences( + tokens: any[], + contentScriptId: string, + Token +) { + DFS(tokens, contentScriptId, Token); +} + +let pattern: RegExp = /\[-@(\w|:|\?|\-)+\]/g; +function DFS(nodes: any[], contentScriptId: string, Token: any): void { + for (let i = 0; i < nodes.length; i++) { + if (nodes[i].type === "text") { + // If it's the default format, keep it as it is + if ( + i - 1 >= 0 && + nodes[i - 1].type === "link_open" && + i + 1 < nodes.length && + nodes[i + 1].type === "link_close" + ) { + continue; + } + + const matches: string[] = nodes[i].content.match(pattern); + if (matches && matches.length) { + const textToken = new Token("text", "", 0); + textToken.content = "(Loading)"; + nodes[i] = textToken; + console.log(nodes[i]); + } + + // Get all the matches of the pattern and replace them with the formatted reference + /* const content: string = nodes[i].content; + const matches = content.match(pattern); + if (matches && matches.length) { + for (let j = 0; j < matches.length; j++) { + const match = matches[j]; + const refId = match.substring(3, match.length - 1); + const script: string = ` + webviewApi.postMessage("${contentScriptId}", ${JSON.stringify( + { + type: GET_REFERENCE_BY_ID, + refId, + } + )}).then(reference => { + const yearView = document.getElementById("bibtex-year-${j}"); + if (reference && reference.year) { + yearView.textContent = "(" + reference.year + ")"; + } + + }); + return false; + `; + + const html: string = ` + (Loading...) +