Local genesis file

.dockerignore

@@ -3,4 +3,9 @@ Dockerfile
 LICENSE
 manage
 README.md
+COMPLIANCE.yaml
 .git*
+tmp/
+backup/
+openshift/
+docs/

.github/workflows/build-docker.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - master # will be renaming soon.
 
 # Ensure only the most recent workflow runs.
 concurrency:

.gitignore

@@ -11,6 +11,9 @@ backup/
 *_BuildConfig.json
 tmp/
 
+# External scripts
+genesis_from_files.py
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Dockerfile

@@ -1,4 +1,4 @@
-FROM bcgovimages/von-image:node-1.12-3
+FROM bcgovimages/von-image:node-1.12-4
 
 ENV LOG_LEVEL ${LOG_LEVEL:-info}
 ENV RUST_LOG ${RUST_LOG:-warning}

README.md

@@ -1,3 +1,6 @@
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
+[![Lifecycle:Stable](https://img.shields.io/badge/Lifecycle-Stable-97ca00)](https://github.com/bcgov/repomountie/blob/master/doc/lifecycle-badges.md)
+
 # VON Network
 
 A portable development level Indy Node network, including a Ledger Browser. The Ledger Browser (for example the BC Gov's [Ledger for the GreenLight Demo Application](http://greenlight.bcovrin.vonx.io/)) allows a user to see the status of the nodes of a network and browse/search/filter the Ledger Transactions.
@@ -28,6 +31,8 @@ Want to see a full demo that includes applications and verifiable credentials be
 
 This repository includes a fully containerized Indy-Cli environment, allowing you to use the Indy-Cli without having to build or install the Indy-SDK or any of its dependencies on your machine.
 
+For more information refer to [Using the containerized `indy-cli`](./docs/Indy-CLI.md)
+
 ## Ledger Troubleshooting
 
 Refer to the [Troubleshooting](./docs/Troubleshooting.md) document for some tips and tools you can use to troubleshoot issues with a ledger.
@@ -45,13 +50,31 @@ The [tutorial about using VON Network](docs/UsingVONNetwork.md) has information
 
 1. Run docker to start the ledger, and pass in GENESIS_URL and LEDGER_SEED parameters:
 
+GENESI_URL could be a http(s) URL or path to genesis file inside the container. 
+
+To use the path to genesis file, you need to use the protocol `file://` in GENESIS_URL and the genesis file need to be inside the container. You have two options:
+
+- Attach a volumen (f.e ./tmp:/tmp) to `webserver`  service in volume section inside `docker-compose.yml` file.
+- Copy the file inside container using `docker cp` .
+
 For example to connect to the Sovrin Test Network:
 
 ```bash
 ./manage build
-GENESIS_URL=https://raw.githubusercontent.com/sovrin-foundation/sovrin/master/sovrin/pool_transactions_sandbox_genesis ./manage start-web
+GENESIS_URL=https://raw.githubusercontent.com/sovrin-foundation/sovrin/master/sovrin/pool_transactions_sandbox_genesis
+LEDGER_SEED=00000000000000000000000000000012
+./manage start-web
 ```
 
+```bash
+./manage build
+GENESIS_URL=file:///tmp/pool_transactions_sandbox_genesis
+LEDGER_SEED=00000000000000000000000000000012
+./manage start-web
+```
+
+
+
 Note that it takes some time to get the transactions and status from the network. Once the UI appears, try getting the `Genesis Transaction` that the server started up properly.
 
 ## Running the web server on your local machine
@@ -136,9 +159,9 @@ GENESIS_URL=https://raw.githubusercontent.com/sovrin-foundation/sovrin/master/so
 3. Download this repository:
 
     ```bash
-    curl -L https://github.com/bcgov/von-network/archive/master.zip > bcovrin.zip && \
+    curl -L https://github.com/bcgov/von-network/archive/main.zip > bcovrin.zip && \
         unzip bcovrin.zip && \
-        cd von-network-master && \
+        cd von-network-main && \
         chmod a+w ./server/
     ```
 

bin/read_ledger

@@ -1,24 +1,30 @@
 #! /usr/bin/env python3
 """
 Convenience script for reading a ledger transaction data (stored in leveldb)
+
 """
 import argparse
 import logging
 import os
 import shutil
 
+from pathlib import Path
+
 from common.serializers.json_serializer import JsonSerializer
-from ledger.compact_merkle_tree import CompactMerkleTree
 from plenum.common.ledger import Ledger
-from storage.helper import initHashStore
+from plenum.common.constants import HS_ROCKSDB
 
 from indy_common.config_util import getConfig
-from indy_common.config_helper import ConfigHelper, NodeConfigHelper
+from common.serializers.serialization import ledger_txn_serializer
 
 logging.root.handlers = []
 logger = logging.getLogger()
 logger.propagate = False
 logger.disabled = True
+_DATA = 'data'
+
+# TODO: Replace with constant from config
+postfix = '_transactions'
 
 
 def read_args():
@@ -39,73 +45,69 @@ def read_args():
     parser.add_argument('--count', required=False, action='store_true',
                         help="returns the number of txns in the given ledger")
     parser.add_argument('--node_name', required=False, help="Node's name")
-    parser.add_argument('--client_name', required=False, help="Client's name")
     parser.add_argument('--serializer', required=False, default='json',
                         help="How to represent the data (json by default)")
     parser.add_argument('--network', required=False, type=str,
                         help="Network name to read ledger from")
-    parser.add_argument('--base_dir', required=False, type=str,
-                        help="Specifify a directory to read from")
 
     return parser.parse_args()
 
 
-def get_ledger_dir(node_name, client_name, network, base_dir):
-
-    if node_name and client_name:
-        print("Either 'node_name' or 'client_name' can be specified")
-        exit()
-
+def get_ledger_dir(node_name, network):
     config = getConfig()
     _network = network if network else config.NETWORK_NAME
-    if node_name or client_name:
-        if node_name:
-            config_helper = NodeConfigHelper(node_name, config)
-            ledger_data_dir = config_helper.ledger_dir
-        else:
-            ledger_data_dir = os.path.join(config.CLI_BASE_DIR, _network,
-                                           config.clientDataDir, client_name)
-        if not os.path.isdir(ledger_data_dir):
-            print("Specified Node or Client folder not found: {}".format(
-                ledger_data_dir))
-            exit()
+    ledger_base_dir = config.LEDGER_DIR
+    if node_name:
+        # Build path to data if --node_name was specified
+        ledger_data_dir = os.path.join(ledger_base_dir, _network, _DATA, node_name)
     else:
-
-        if not base_dir:
-            base_dir = config.CLI_BASE_DIR if client_name else config.baseDir
-        data_dir = config.clientDataDir if client_name else 'data'
-        ledger_data_dir = os.path.join(base_dir, _network, data_dir)
-
-        if not os.path.exists(ledger_data_dir):
-            print("Can not find the directory with the ledger: {}".format(
-                ledger_data_dir))
-            exit()
-
-        dirs = os.listdir(ledger_data_dir)
-        ledger_data_dir = os.path.join(ledger_data_dir, dirs[0])
+        ledger_data_dir = os.path.join(ledger_base_dir, _network, _DATA)
+        if os.path.exists(ledger_data_dir):
+            dirs = os.listdir(ledger_data_dir)
+            if len(dirs) == 0:
+                print("Node's 'data' folder not found: {}".format(ledger_data_dir))
+                exit()
+            # --node_name parameter was not set, therefore we can choose first Node name in data dir
+            ledger_data_dir = os.path.join(ledger_data_dir, dirs[0])
+    if not os.path.exists(ledger_data_dir):
+        print("No such file or directory: {}".format(ledger_data_dir))
+        print("Please check, that network: '{}' was used ".format(_network))
+        exit()
 
     return ledger_data_dir
 
 
-def get_ledger(type_, ledger_data_dir):
+def get_storage(type_, ledger_data_dir):
     config = getConfig()
 
-    ledger_name = None
+    storage_name = None
     if type_ == 'pool':
-        ledger_name = config.poolTransactionsFile
+        storage_name = config.poolTransactionsFile
     elif type_ == 'domain':
-        ledger_name = config.domainTransactionsFile
+        storage_name = config.domainTransactionsFile
     elif type_ == 'config':
-        ledger_name = config.configTransactionsFile
+        storage_name = config.configTransactionsFile
+    elif type_ in get_additional_storages(ledger_data_dir):
+        storage_name = type_ + postfix
     else:
         print("Unknown ledger type: {}".format(type_))
         exit()
 
-    hash_store = initHashStore(ledger_data_dir, type_, config)
-    return Ledger(CompactMerkleTree(hashStore=hash_store), dataDir=ledger_data_dir, fileName=ledger_name)
+    return Ledger._defaultStore(dataDir=ledger_data_dir,
+                                logName=storage_name,
+                                ensureDurability=True,
+                                open=True,
+                                config=config,
+                                read_only=True)
+
 
+def get_additional_storages(ledger_data_dir):
+    additional_storages = \
+        [name[:name.find(postfix)] for name in os.listdir(ledger_data_dir) if postfix in name]
+    return additional_storages
 
-def print_txns(ledger, args):
+
+def print_txns(storage, args):
     serializer = None
     if args.serializer == 'json':
         serializer = JsonSerializer()
@@ -116,40 +118,43 @@ def print_txns(ledger, args):
     # --count
     count = args.count
     if count:
-        print_count(ledger)
+        print_count(storage)
         return
 
     # --seq_no
-    seq_no = int(args.seq_no) if args.seq_no is not None else None
+    seq_no = args.seq_no
     if seq_no:
-        print_by_seq_no(ledger, seq_no, serializer)
+        print_by_seq_no(storage, seq_no, serializer)
         return
 
     # print all (--from --to)
-    print_all(ledger, serializer)
+    print_all(storage, serializer)
 
 
-def print_by_seq_no(ledger, seq_no, serializer):
+def print_by_seq_no(storage, seq_no, serializer):
     try:
-        txn = ledger.getBySeqNo(seq_no)
+        print("Getting by seq no:", seq_no)
+        txn = storage.get(seq_no)
     except KeyError:
         print('No transactions found for seq_no={}'.format(seq_no))
         return
+    print("Serializing transactions ...")
     txn = serializer.serialize(txn, toBytes=False)
+    print("Printing transactions ...")
     print(txn)
     return
 
 
-def print_count(ledger):
-    print(ledger.size)
+def print_count(storage):
+    print(storage.size)
 
 
-def print_all(ledger, serializer):
-    frm = int(args.frm) if args.frm is not None else None
-    to = int(args.to) if args.to is not None else None
-    for txn in ledger.getAllTxn(frm=frm, to=to):
-        txn = serializer.serialize(txn, toBytes=False)
-        print(txn)
+def print_all(storage, serializer):
+    frm = int(args.frm) if args.frm else None
+    to = int(args.to) if args.to else None
+    for seqNo, txn in storage.iterator(start=frm, end=to):
+        txn = ledger_txn_serializer.deserialize(txn)
+        print(serializer.serialize(txn, toBytes=False))
 
 
 def make_copy_of_ledger(data_dir):
@@ -162,20 +167,30 @@ def make_copy_of_ledger(data_dir):
 
 if __name__ == '__main__':
     args = read_args()
+    config = getConfig()
 
-    # TODO: works well only for small ledgers,
-    ledger_data_dir = get_ledger_dir(
-        args.node_name,
-        args.client_name,
-        args.network,
-        args.base_dir)
-
+    ledger_data_dir = get_ledger_dir(args.node_name, args.network)
     read_copy_ledger_data_dir = None
     try:
-        read_copy_ledger_data_dir = make_copy_of_ledger(ledger_data_dir)
-        ledger = get_ledger(args.type, read_copy_ledger_data_dir)
-        print_txns(ledger, args)
+        # RocksDB supports real read-only mode and does not need to have a ledger copy.
+        if config.hashStore['type'].lower() != HS_ROCKSDB:
+            config.db_transactions_config = None
+            # NOTE: such approach works well only for small ledgers.
+            tmp = make_copy_of_ledger(ledger_data_dir)
+
+            # Let's be paranoid to avoid removing of ledger instead of its copy.
+            ledger_path = Path(ledger_data_dir)
+            ledger_copy_path = Path(tmp)
+            assert ledger_path != ledger_copy_path
+            assert ledger_copy_path not in ledger_path.parents
+
+            read_copy_ledger_data_dir = tmp
+            ledger_data_dir = read_copy_ledger_data_dir
+        elif config.db_transactions_config is not None:
+            # This allows to avoid debug logs creation on each read_ledger run
+            config.db_transactions_config['db_log_dir'] = '/dev/null'
+        storage = get_storage(args.type, ledger_data_dir)
+        print_txns(storage, args)
     finally:
-        # TODO be careful about removing original the ledger data dir
         if read_copy_ledger_data_dir:
             shutil.rmtree(read_copy_ledger_data_dir)

cli-scripts/generate_did.py

@@ -0,0 +1,38 @@
+import os
+import logging
+import base64
+import argparse
+import asyncio
+import sys
+# Import server.anchor from the path relative to where the scripts are being executed.
+sys.path.insert(1, './server')
+from anchor import AnchorHandle
+
+logging.getLogger().setLevel(logging.ERROR)
+
+async def generate_did(seed):
+    TRUST_ANCHOR = AnchorHandle()
+    did, verkey = await TRUST_ANCHOR.seed_to_did(seed)
+    print(f"\nSeed: {seed}")
+    print(f"DID: {did}")
+    print(f"Verkey: {verkey}")
+
+def main(seed):
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(generate_did(seed))
+    loop.close()
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Generates a DID and Verkey from a Seed.")
+    parser.add_argument("--seed", required=True, default=os.environ.get('SEED'), help="The seed to use to generate the DID and Verkey.")
+    args, unknown = parser.parse_known_args()
+
+    testseed = args.seed
+    if testseed.endswith("="):
+        testseed = base64.b64decode(testseed).decode("ascii")
+
+    if not len(testseed) == 32:
+        print("Seed must be 32 characters long.")
+        exit()
+
+    main(args.seed)

docker-compose.yml

@@ -47,6 +47,7 @@ services:
     volumes:
       - ./config:/home/indy/config
       - ./server:/home/indy/server
+      - ./tmp:/tmp
       - webserver-cli:/home/indy/.indy-cli
       - webserver-ledger:/home/indy/ledger