From 94c7f55e3c81dd30b62d1af1a9b8c207200645ce Mon Sep 17 00:00:00 2001
From: Elliot Silver <elliot.silver@resmed.com>
Date: Mon, 2 Feb 2026 16:20:48 -0800
Subject: [PATCH 1/3] don't try showing content if module is not configured

---
 server.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/server.js b/server.js
index b542468..74f4890 100644
--- a/server.js
+++ b/server.js
@@ -221,28 +221,28 @@ async function buildRootPageContent() {
   content += '<ul class="list-group">';
 
   // Check which modules are enabled and add them to the list
-  if (config.modules.packages.enabled) {
+  if (config.modules.packages && config.modules.packages.enabled) {
     mc++;
     content += '<li class="list-group-item">';
     content += '<a href="/packages" class="text-decoration-none">Package Server</a>: Browse and download FHIR Implementation Guide packages';
     content += '</li>';
   }
 
-  if (config.modules.xig.enabled) {
+  if (config.modules.xig && config.modules.xig.enabled) {
     mc++;
     content += '<li class="list-group-item">';
     content += '<a href="/xig" class="text-decoration-none">FHIR IG Statistics</a>: Statistics and analysis of FHIR Implementation Guides';
     content += '</li>';
   }
 
-  if (config.modules.shl.enabled) {
+  if (config.modules.shl && config.modules.shl.enabled) {
     mc++;
     content += '<li class="list-group-item">';
     content += '<a href="/shl" class="text-decoration-none">SHL Server</a>: SMART Health Links management and validation';
     content += '</li>';
   }
 
-  if (config.modules.vcl.enabled) {
+  if (config.modules.vcl && config.modules.vcl.enabled) {
     mc++;
     content += '<li class="list-group-item">';
     content += '<a href="/VCL" class="text-decoration-none">VCL Server</a>: ValueSet Compose Language expression parsing';

From 29e3f7d6fe663f4a9eb97c111b8daf51c10e8a24 Mon Sep 17 00:00:00 2001
From: Elliot Silver <elliot.silver@resmed.com>
Date: Mon, 2 Feb 2026 16:24:05 -0800
Subject: [PATCH 2/3] handle never crawled packages

---
 library/html-server.js | 1 +
 packages/packages.js   | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/library/html-server.js b/library/html-server.js
index 5850429..629fb4a 100644
--- a/library/html-server.js
+++ b/library/html-server.js
@@ -84,6 +84,7 @@ class HtmlServer {
       .replace(/\[%content%\]/g, content) // Content is assumed to be already-safe HTML
       .replace(/\[%ver%\]/g, this.escapeHtml(renderOptions.version))
       .replace(/\[%download-date%\]/g, this.escapeHtml(renderOptions.downloadDate))
+      .replace(/\[%crawler-date%\]/g, this.escapeHtml(renderOptions.crawlerDate || renderOptions.downloadDate || 'Never'))
       .replace(/\[%total-resources%\]/g, this.escapeHtml(renderOptions.totalResources.toLocaleString()))
       .replace(/\[%total-packages%\]/g, this.escapeHtml(renderOptions.totalPackages.toLocaleString()))
       .replace(/\[%endpoint-path%\]/g, this.escapeHtml(renderOptions.endpointpath))
diff --git a/packages/packages.js b/packages/packages.js
index 03c71c2..1388723 100644
--- a/packages/packages.js
+++ b/packages/packages.js
@@ -421,8 +421,15 @@ class PackagesModule {
       // Get counts from database
       const tableCounts = await this.getDatabaseTableCounts();
 
+      // Format crawler date
+      let crawlerDate = 'Never';
+      if (this.lastRunTime) {
+        crawlerDate = new Date(this.lastRunTime).toLocaleDateString();
+      }
+
       return {
         downloadDate: downloadDate,
+        crawlerDate: crawlerDate,
         totalResources: 0, // Packages don't track individual resources
         totalPackages: tableCounts.packages || 0,
         totalVersions: tableCounts.packageVersions || 0,
@@ -437,6 +444,7 @@ class PackagesModule {
 
       return {
         downloadDate: 'Error',
+        crawlerDate: 'Never',
         totalResources: 0,
         totalPackages: 0,
         totalVersions: 0,

From 705b50f772ea338035411b97ae3992274ef01ede Mon Sep 17 00:00:00 2001
From: Elliot Silver <elliot.silver@resmed.com>
Date: Mon, 2 Feb 2026 16:50:33 -0800
Subject: [PATCH 3/3] support masterFile as alternative to masterUrl

---
 packages/package-crawler.js | 36 +++++++++++++++++++++++++++------
 packages/packages.js        | 40 +++++++++++++++++++++++++++----------
 2 files changed, 60 insertions(+), 16 deletions(-)

diff --git a/packages/package-crawler.js b/packages/package-crawler.js
index d80c429..6e05a7b 100644
--- a/packages/package-crawler.js
+++ b/packages/package-crawler.js
@@ -27,19 +27,20 @@ class PackageCrawler {
     this.log = log;
     
     const startTime = Date.now();
+    const masterSource = this.config.masterFile || this.config.masterUrl;
     this.crawlerLog = {
       startTime: new Date().toISOString(),
-      master: this.config.masterUrl,
+      master: masterSource,
       feeds: [],
       totalBytes: 0,
       errors: ''
     };
 
-    this.log.info('Running web crawler for packages using master URL: '+ this.config.masterUrl);
+    this.log.info('Running web crawler for packages using master source: '+ masterSource);
 
     try {
-      // Fetch the master JSON file
-      const masterResponse = await this.fetchJson(this.config.masterUrl);
+      // Fetch the master JSON file using unified fetchJson method
+      const masterResponse = await this.fetchJson(masterSource);
 
       if (!masterResponse.feeds || !Array.isArray(masterResponse.feeds)) {
         throw new Error('Invalid master JSON: missing feeds array');
@@ -58,7 +59,7 @@ class PackageCrawler {
         try {
           await this.updateTheFeed(
             this.fixUrl(feedConfig.url),
-            this.config.masterUrl,
+            masterSource,
             feedConfig.errors ? feedConfig.errors.replace(/\|/g, '@').replace(/_/g, '.') : '',
             packageRestrictions
           );
@@ -93,7 +94,30 @@ class PackageCrawler {
     return url.replace(/^http:/, 'https:');
   }
 
-  async fetchJson(url) {
+  async fetchJson(source) {
+    // Determine if source is a file path or URL
+    if (this.isFilePath(source)) {
+      return await this.fetchJsonFromFile(source);
+    } else {
+      return await this.fetchJsonFromUrl(source);
+    }
+  }
+
+  isFilePath(source) {
+    // Check if it's a file path (not a URL)
+    return !source.startsWith('http://') && !source.startsWith('https://');
+  }
+
+  async fetchJsonFromFile(filePath) {
+    try {
+      const data = await fs.promises.readFile(filePath, 'utf8');
+      return JSON.parse(data);
+    } catch (error) {
+      throw new Error(`Failed to read JSON from file ${filePath}: ${error.message}`);
+    }
+  }
+
+  async fetchJsonFromUrl(url) {
     try {
       const response = await axios.get(url, {
         timeout: 30000,
diff --git a/packages/packages.js b/packages/packages.js
index 1388723..4c1fd1b 100644
--- a/packages/packages.js
+++ b/packages/packages.js
@@ -563,10 +563,24 @@ class PackagesModule {
   async initialize(config) {
     this.config = config;
 
-    // Set default masterUrl if not configured
-    if (!this.config.masterUrl) {
+    // Validate masterFile/masterUrl configuration
+    if (this.config.masterFile && this.config.masterUrl) {
+      throw new Error('Cannot specify both masterFile and masterUrl. Please use only one.');
+    }
+
+    // Process masterFile if specified - normalize path early
+    if (this.config.masterFile) {
+      // If not absolute path, resolve relative to data directory
+      if (!path.isAbsolute(this.config.masterFile)) {
+        this.config.masterFile = folders.filePath(this.config.masterFile);
+      }
+      pckLog.info(`Using masterFile: ${this.config.masterFile}`);
+    }
+
+    // Set default masterUrl if neither masterFile nor masterUrl configured
+    if (!this.config.masterFile && !this.config.masterUrl) {
       this.config.masterUrl = 'https://fhir.github.io/ig-registry/package-feeds.json';
-      pckLog.info('No masterUrl configured, using default:', this.config.masterUrl);
+      pckLog.info('No masterFile or masterUrl configured, using default masterUrl:', this.config.masterUrl);
     }
 
     pckLog.info('Initializing Packages module...');
@@ -834,21 +848,22 @@ class PackagesModule {
   async runWebCrawler() {
     const startTime = Date.now();
     this.totalRuns++;
+    const masterSource = this.config.masterFile || this.config.masterUrl;
     this.crawlerLog = {
       runNumber: this.totalRuns,
       startTime: new Date().toISOString(),
-      master: this.config.masterUrl,
+      master: masterSource,
       feeds: [],
       totalBytes: 0,
       errors: ''
     };
 
     pckLog.info(`Running web crawler for packages (run #${this.totalRuns})...`);
-    pckLog.info('Fetching master URL:', this.config.masterUrl);
+    pckLog.info('Fetching master from:', masterSource);
 
     try {
-      // Fetch the master JSON file
-      const masterResponse = await this.fetchJson(this.config.masterUrl);
+      // Fetch the master JSON file using crawler's unified fetchJson method
+      const masterResponse = await this.crawler.fetchJson(masterSource);
 
       if (!masterResponse.feeds || !Array.isArray(masterResponse.feeds)) {
         throw new Error('Invalid master JSON: missing feeds array');
@@ -867,7 +882,7 @@ class PackagesModule {
         try {
           await this.updateTheFeed(
             this.fixUrl(feedConfig.url),
-            this.config.masterUrl,
+            masterSource,
             feedConfig.errors ? feedConfig.errors.replace(/\|/g, '@').replace(/_/g, '.') : '',
             packageRestrictions
           );
@@ -1220,7 +1235,8 @@ class PackagesModule {
               mirror: this.config.mirrorPath
             },
             config: {
-              masterUrl: this.config.masterUrl
+              masterUrl: this.config.masterUrl,
+              masterFile: this.config.masterFile
             }
           });
         }
@@ -2767,7 +2783,11 @@ class PackagesModule {
     if (this.lastRunTime) {
       content += `<tr><td>Last Run</td><td>${new Date(this.lastRunTime).toLocaleString()}</td></tr>`;
     }
-    content += `<tr><td>Master URL</td><td><a href="${htmlServer.escapeHtml(this.config.masterUrl)}" target="_blank">${htmlServer.escapeHtml(this.config.masterUrl)}</a></td></tr>`;
+    if (this.config.masterFile) {
+      content += `<tr><td>Master File</td><td>${htmlServer.escapeHtml(this.config.masterFile)}</td></tr>`;
+    } else {
+      content += `<tr><td>Master URL</td><td><a href="${htmlServer.escapeHtml(this.config.masterUrl)}" target="_blank">${htmlServer.escapeHtml(this.config.masterUrl)}</a></td></tr>`;
+    }
     content += '</table>';
     content += '</div>';
     content += '</div>';