Skip to content

Add census --live for real-time ecosystem data collection #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thebenignhacker merged 1 commit into from
Mar 18, 2026
Merged

Add census --live for real-time ecosystem data collection #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions docs/cli.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,10 +195,21 @@ CRYPTOSERVE > gate
Analyze cryptographic library adoption across package ecosystems.

```bash
cryptoserve census # Run ecosystem census
cryptoserve census --ecosystems npm,pypi # Specific ecosystems
cryptoserve census # Run ecosystem census (cached/offline data)
cryptoserve census --live # Fetch real-time data from npm, PyPI, crates.io
cryptoserve census --live --ecosystems npm # Query only npm
cryptoserve census --live --format json # Machine-readable JSON output
```

| Flag | Description |
|------|-------------|
| `--live` | Fetch real-time download counts from package registries (npm, PyPI, crates.io) |
| `--ecosystems <list>` | Comma-separated list of ecosystems to query: `npm`, `pypi`, `crates` (default: all three) |
| `--format <fmt>` | Output format: `text` (default), `json`, `html` |
| `--output <file>` | Write output to a file |
| `--no-cache` | Skip cached data |
| `--verbose` | Show detailed progress |

---

## Offline Crypto Tools (no server required)
Expand Down
263 changes: 260 additions & 3 deletions sdk/javascript/bin/cryptoserve.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
* cryptoserve login [--server URL]
* cryptoserve status
* cryptoserve census [--format json|html] [--output file] [--no-cache] [--verbose]
* cryptoserve census --live [--ecosystems npm,pypi,crates] [--format json]
*/

import { readFileSync, writeFileSync } from 'node:fs';
Expand All @@ -37,11 +38,12 @@ const PKG = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf-
const OPTIONS_WITH_VALUES = new Set([
'--password', '--algorithm', '--profile', '--format', '--file',
'--output', '--server', '--context', '--max-risk', '--min-score',
'--ecosystems',
]);

const KNOWN_FLAGS = new Set([
'--insecure-storage', '--verbose', '--binary', '--fail-on-weak',
'--help', '--version', '--no-cache',
'--help', '--version', '--no-cache', '--live',
]);

function getFlag(args, name) {
Expand Down Expand Up @@ -93,7 +95,7 @@ async function cmdHelp() {
console.log(` ${info('gate [path] [--max-risk R]')} CI/CD gate (exit 0=pass, 1=fail)`);
console.log();
console.log(` ${bold('Research')}`);
console.log(` ${info('census [--format json|html]')} Global crypto census (11 ecosystems + NVD)`);
console.log(` ${info('census [--live] [--format json|html]')} Global crypto census (11 ecosystems + NVD)`);
console.log();
console.log(` ${bold('Encryption')}`);
console.log(` ${info('encrypt "text" [--context C]')} Encrypt with context-aware algorithm selection`);
Expand Down Expand Up @@ -1010,7 +1012,262 @@ function timeSince(date) {
// Census — global crypto adoption survey
// ---------------------------------------------------------------------------

/**
* Census --live: fetch real-time download data from npm, PyPI, crates.io.
*/
async function cmdCensusLive(args) {
const {
compactHeader, section, labelValue, tableHeader, tableRow,
warning, info, dim, bold, divider,
} = await import('../lib/cli-style.mjs');
const { formatNumber } = await import('../lib/census/aggregator.mjs');
const {
NPM_PACKAGES, PYPI_PACKAGES, CRATES_PACKAGES, TIERS,
} = await import('../lib/census/package-catalog.mjs');

const format = getOption(args, '--format', 'text');
const ecosystemArg = getOption(args, '--ecosystems', 'npm,pypi,crates');
const enabledEcosystems = ecosystemArg.split(',').map(e => e.trim().toLowerCase());

const ECOSYSTEM_CONFIG = {
npm: { label: 'npm', packages: NPM_PACKAGES, delay: 100 },
pypi: { label: 'PyPI', packages: PYPI_PACKAGES, delay: 2000 },
crates: { label: 'crates.io', packages: CRATES_PACKAGES, delay: 200 },
};

const sleep = (ms) => new Promise(r => setTimeout(r, ms));

// --- Fetch functions ---

async function fetchNpm(pkg) {
try {
const res = await fetch(`https://api.npmjs.org/downloads/point/last-month/${pkg.name}`);
if (!res.ok) return 0;
const data = await res.json();
return data.downloads || 0;
} catch { return 0; }
}

async function fetchPypi(pkg) {
try {
const res = await fetch(`https://pypistats.org/api/packages/${pkg.name}/recent`);
if (!res.ok) return 0;
const data = await res.json();
return data?.data?.last_month || 0;
} catch { return 0; }
}

async function fetchCrates(pkg) {
try {
const res = await fetch(`https://crates.io/api/v1/crates/${pkg.name}`, {
headers: { 'User-Agent': 'cryptoserve-census/1.0 (https://cryptoserve.dev)' },
});
if (!res.ok) return 0;
const data = await res.json();
const recent = data?.crate?.recent_downloads || 0;
return Math.round(recent / 3); // 90-day -> monthly estimate
} catch { return 0; }
}

const fetchFns = { npm: fetchNpm, pypi: fetchPypi, crates: fetchCrates };

// --- Collect data ---

if (format === 'text') {
console.log(compactHeader('census --live'));
console.log('');
console.log(dim(' Collecting live data from package registries...'));
console.log('');
}

const ecosystemResults = {};

for (const ecoKey of enabledEcosystems) {
const config = ECOSYSTEM_CONFIG[ecoKey];
if (!config) {
if (format === 'text') {
console.log(warning(` Unknown ecosystem: ${ecoKey} (supported: npm, pypi, crates)`));
}
continue;
}

const fetchFn = fetchFns[ecoKey];
const packages = config.packages;
const results = [];

if (format === 'text') {
process.stdout.write(` ${config.label} (${packages.length} packages)`.padEnd(32));
}

for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
const downloads = await fetchFn(pkg);
results.push({
name: pkg.name,
downloads,
tier: pkg.tier,
category: pkg.category,
ecosystem: ecoKey,
});
if (i < packages.length - 1) {
await sleep(config.delay);
}
}

if (format === 'text') {
console.log('done');
}

ecosystemResults[ecoKey] = {
label: config.label,
packages: results.sort((a, b) => b.downloads - a.downloads),
packageCount: packages.length,
};
}

// --- Compute tier breakdowns ---

const ecosystemSummaries = [];
let grandTotal = 0;
let grandWeak = 0;
let grandModern = 0;
let grandPqc = 0;
const allPackages = [];

for (const [ecoKey, eco] of Object.entries(ecosystemResults)) {
let weak = 0, modern = 0, pqc = 0;
for (const pkg of eco.packages) {
if (pkg.tier === TIERS.WEAK) weak += pkg.downloads;
else if (pkg.tier === TIERS.PQC) pqc += pkg.downloads;
else modern += pkg.downloads;
allPackages.push(pkg);
}
const total = weak + modern + pqc;
ecosystemSummaries.push({
key: ecoKey,
label: eco.label,
total,
weak,
modern,
pqc,
weakPct: total > 0 ? (weak / total * 100) : 0,
modernPct: total > 0 ? (modern / total * 100) : 0,
pqcPct: total > 0 ? (pqc / total * 100) : 0,
});
grandTotal += total;
grandWeak += weak;
grandModern += modern;
grandPqc += pqc;
}

// Top 5 weak packages
const topWeak = allPackages
.filter(p => p.tier === TIERS.WEAK && p.downloads > 0)
.sort((a, b) => b.downloads - a.downloads)
.slice(0, 5);

// NIST 2030 deadline
const nist2030 = new Date('2030-01-01T00:00:00Z');
const daysLeft = Math.max(0, Math.ceil((nist2030.getTime() - Date.now()) / (1000 * 60 * 60 * 24)));
const yearsLeft = Math.floor(daysLeft / 365);
const monthsLeft = Math.floor((daysLeft % 365) / 30);
const nistLabel = `${yearsLeft}y ${monthsLeft}mo remaining`;

// --- JSON output ---

if (format === 'json') {
const jsonOutput = {
command: 'census --live',
collectedAt: new Date().toISOString(),
ecosystems: {},
totals: {
downloads: grandTotal,
weak: grandWeak,
modern: grandModern,
pqc: grandPqc,
weakPercentage: grandTotal > 0 ? Math.round(grandWeak / grandTotal * 1000) / 10 : 0,
modernPercentage: grandTotal > 0 ? Math.round(grandModern / grandTotal * 1000) / 10 : 0,
pqcPercentage: grandTotal > 0 ? Math.round(grandPqc / grandTotal * 1000) / 10 : 0,
},
topWeakPackages: topWeak.map(p => ({
name: p.name,
ecosystem: p.ecosystem,
downloads: p.downloads,
})),
nist2030Deadline: { daysRemaining: daysLeft, label: nistLabel },
};
for (const summary of ecosystemSummaries) {
const eco = ecosystemResults[summary.key];
jsonOutput.ecosystems[summary.key] = {
label: summary.label,
packageCount: eco.packageCount,
totalDownloads: summary.total,
weak: summary.weak,
modern: summary.modern,
pqc: summary.pqc,
weakPercentage: Math.round(summary.weakPct * 10) / 10,
modernPercentage: Math.round(summary.modernPct * 10) / 10,
pqcPercentage: Math.round(summary.pqcPct * 10) / 10,
packages: eco.packages,
};
}
console.log(JSON.stringify(jsonOutput, null, 2));
return;
}

// --- Terminal table output ---

console.log('');
const colWidths = [14, 14, 10, 10, 8];
console.log(tableHeader(['Ecosystem', 'Total/mo', 'Weak %', 'Modern %', 'PQC %'], colWidths));

for (const s of ecosystemSummaries) {
console.log(tableRow([
s.label,
formatNumber(s.total),
s.weakPct.toFixed(1) + '%',
s.modernPct.toFixed(1) + '%',
s.pqcPct.toFixed(1) + '%',
], colWidths));
}

if (ecosystemSummaries.length > 1) {
console.log(divider(56));
const grandWeakPct = grandTotal > 0 ? (grandWeak / grandTotal * 100) : 0;
const grandModernPct = grandTotal > 0 ? (grandModern / grandTotal * 100) : 0;
const grandPqcPct = grandTotal > 0 ? (grandPqc / grandTotal * 100) : 0;
console.log(tableRow([
'Total',
formatNumber(grandTotal),
grandWeakPct.toFixed(1) + '%',
grandModernPct.toFixed(1) + '%',
grandPqcPct.toFixed(1) + '%',
], colWidths));
}

console.log('');

if (topWeak.length > 0) {
console.log(section('Top 5 Weak Packages'));
const ecoLabels = { npm: 'npm', pypi: 'PyPI', crates: 'crates' };
for (let i = 0; i < topWeak.length; i++) {
const p = topWeak[i];
const ecoLabel = ecoLabels[p.ecosystem] || p.ecosystem;
console.log(` ${i + 1}. ${p.name} (${ecoLabel})`.padEnd(40) + `${formatNumber(p.downloads)}/mo`);
}
console.log('');
}

console.log(dim(` NIST 2030 Deadline: ${nistLabel}`));
console.log('');
}

async function cmdCensus(args) {
if (getFlag(args, '--live')) {
await cmdCensusLive(args);
return;
}

const {
compactHeader, section, labelValue, tableHeader, tableRow,
warning, info, dim, bold, divider, progressBar,
Expand Down Expand Up @@ -1076,7 +1333,7 @@ const COMMAND_HELP = {
vault: 'cryptoserve vault init|set|get|list|delete|run|import|export|reset [--password P]\n\n Manage an encrypted secrets vault.\n Use --password for non-interactive/CI usage.',
login: 'cryptoserve login [--server URL]\n\n Authenticate with a CryptoServe server.',
status: 'cryptoserve status\n\n Show configuration and server connection status.',
census: 'cryptoserve census [--format json|html] [--output file] [--no-cache] [--verbose]\n\n Global crypto adoption census across 11 ecosystems + NVD.',
census: 'cryptoserve census [--format json|html] [--output file] [--no-cache] [--verbose]\ncryptoserve census --live [--ecosystems npm,pypi,crates] [--format json]\n\n Global crypto adoption census across 11 ecosystems + NVD.\n --live fetches real-time download data from package registries.\n --ecosystems limits which registries to query (comma-separated: npm,pypi,crates).',
};

function showCommandHelp(command) {
Expand Down
Loading