1 docker image

.github/workflows/production.yml

@@ -32,19 +32,9 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-# TODO
-# this workflow should not build image, but add tag to already existing image.
-# we can't do it right now, because of BUILD_ENV=prod. It needs to be change in the project first.
-
-      - name: ${{ matrix.image }} image
-        uses: docker/build-push-action@v3
+      - name: add tag to ${{ matrix.image }} image
+        uses: abronin/ecr-retag-action@v1
         with:
-          push: true
-          file: docker-builds/${{ matrix.image }}/Dockerfile
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          tags: |
-            ${{ secrets.ECR_REGISTRY }}/namebazaar-${{ matrix.image }}:${{ steps.version.outputs.git_tag }}-prod
-            ${{ secrets.ECR_REGISTRY }}/namebazaar-${{ matrix.image }}:latest-prod
-          build-args: |
-            BUILD_ENV=prod
+          repository: ${{ secrets.ECR_REGISTRY }}/namebazaar-${{ matrix.image }}
+          tag: ${{ github.sha }}
+          new-tags: ${{ steps.version.outputs.git_tag }}

.github/workflows/qa.yml

@@ -38,10 +38,8 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           tags: |
-            ${{ secrets.ECR_REGISTRY }}/namebazaar-${{ matrix.image }}:${{ github.sha }}-qa
-            ${{ secrets.ECR_REGISTRY }}/namebazaar-${{ matrix.image }}:latest-qa
-          build-args: |
-            BUILD_ENV=qa
+            ${{ secrets.ECR_REGISTRY }}/namebazaar-${{ matrix.image }}:${{ github.sha }}
+            ${{ secrets.ECR_REGISTRY }}/namebazaar-${{ matrix.image }}:latest
   trivy:
     needs: build-and-push
     runs-on: ubuntu-latest
@@ -63,7 +61,7 @@ jobs:
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@2a2157eb22c08c9a1fac99263430307b8d1bc7a2
         with:
-          image-ref: '${{ secrets.ECR_REGISTRY }}/namebazaar-${{ matrix.image }}:${{ github.sha }}-qa'
+          image-ref: '${{ secrets.ECR_REGISTRY }}/namebazaar-${{ matrix.image }}:${{ github.sha }}'
           format: 'template'
           template: '@/contrib/sarif.tpl'
           output: 'trivy-results.sarif'

dev/user.clj

@@ -53,13 +53,9 @@
 (defn start-ui!
   "Start the client build.
   Passing {:prod-config? true} points the client to the production server and logging service!"
-  [& {:keys [:prod-config?]}]
-  (let [environment (if prod-config? "prod" "dev")
-        fig-config  (fw-config/fetch-config)]
-    (when prod-config? (log/info "Performing ui-only build..."))
-    (fw-repl/start-figwheel!
-     (set-closure-define fig-config "dev-ui" 'name-bazaar.ui.config.environment environment)
-     "dev-ui")
+  [& {:keys [prod-config?]}]
+  (let [fig-config  (fw-config/fetch-config)]
+    (fw-repl/start-figwheel! "dev-ui")
     (fw-repl/cljs-repl "dev-ui")))
 
 

docker-builds/server/Dockerfile

@@ -28,6 +28,7 @@ RUN . /root/.nvm/nvm.sh \
 FROM env-with-extra-deps AS env-with-clj-deps
 # here instead of "COPY . ." should be "COPY project.clj ." etc. for each file
 # without this big part about cache and speed up build is lost
+# COPY project.clj .
 COPY . .
 RUN lein deps
 RUN lein npm i
@@ -38,13 +39,12 @@ RUN . /root/.nvm/nvm.sh \
     && nvm use "${NODE_VERSION_FOR_TRUFFLE}" \
     && truffle compile \
     && nvm use "${NODE_VERSION}"
-RUN lein build-prod-server
+RUN lein build-server
 
 FROM node:10-buster-slim
 WORKDIR /namebazaar
-ARG BUILD_ENV="prod"
 EXPOSE 6200
-ENV CONFIG="/configs/namebazaar.config.${BUILD_ENV}.edn"
+
 ENV PORT=6200
 RUN apt-get update && \
     apt-get upgrade -yqq && \

docker-builds/ui/Dockerfile

@@ -33,13 +33,12 @@ RUN lein deps
 RUN lein npm i
 
 FROM env-with-clj-deps AS builder
-ARG BUILD_ENV="prod"
 # COPY . .
 RUN . /root/.nvm/nvm.sh \
     && nvm use "${NODE_VERSION_FOR_TRUFFLE}" \
     && truffle compile \
     && nvm use "${NODE_VERSION}"
-RUN lein build-css && lein "build-${BUILD_ENV}-ui"
+RUN lein build-css && lein "build-ui"
 
 FROM builder AS tests
 RUN nginx -t

project.clj

@@ -36,6 +36,7 @@
                  [district0x/bignumber "1.0.3"]
                  [district0x/district-encryption "1.0.1"]
                  [district0x/district-sendgrid "1.0.1"]
+                 [district0x/district-ui-server-config "1.0.0"]
                  [district0x/district-server-config "1.0.1"]
                  [district0x/district-server-db "1.0.4"]
                  ;; TODO: Update to latest version.
@@ -134,15 +135,11 @@
   :auto {"compile-solidity" {:file-pattern #"\.(sol)$"
                              :paths ["resources/public/contracts/src"]}}
 
-  :aliases {"compile-solidity" ["shell" "./compile-solidity.sh"]
-            "clean-prod-server" ["shell" "rm" "-rf" "server"]
-            "watch-css" ["shell" "./semantic.sh" "watch"]
+  :aliases {"watch-css" ["shell" "./semantic.sh" "watch"]
             "build-css" ["shell" "./semantic.sh" "build-css"]
-            "build-prod-server" ["do" ["clean-prod-server"] ["cljsbuild" "once" "server"]]
-            "build-prod-ui" ["do" ["clean"] ["cljsbuild" "once" "min"]]
-            "build-qa-ui" ["do" ["clean"] ["cljsbuild" "once" "qa-min"]]
-            "build-prod" ["pdo" ["build-prod-server"] ["build-prod-ui"] ["build-css"]]
-            "build-qa" ["pdo" ["build-prod-server"] ["build-qa-ui"] ["build-css"]]
+            "clean-server" ["shell" "rm" "-rf" "server"]
+            "build-server" ["do" ["clean-server"] ["cljsbuild" "once" "server"]]
+            "build-ui" ["do" ["clean"] ["cljsbuild" "once" "min"]]
             "run-slither" ["shell" "./run-slither.sh"]}
 
   :profiles {:dev {:dependencies [[org.clojure/clojure "1.9.0"]
@@ -155,23 +152,19 @@
                    :source-paths ["dev" "src"]
                    :resource-paths ["resources"]}}
 
-  :cljsbuild {:builds [;; Development on client-side UI, which uses a testnet
-                       {:id "dev-ui"
+  :cljsbuild {:builds [;; server mainnet
+                       {:id "server"
                         :source-paths ["src"]
-                        :figwheel {:on-jsload "name-bazaar.ui.core/mount-root"}
-                        :compiler {:main "name-bazaar.ui.core"
-                                   :output-to "resources/public/js/compiled/app.js"
-                                   :output-dir "resources/public/js/compiled/out"
-                                   :asset-path "/js/compiled/out"
-                                   :source-map-timestamp true
-                                   :optimizations :none
-                                   :preloads [print.foo.preloads.devtools
-                                              day8.re-frame-10x.preload]
-                                   :closure-defines {name-bazaar.ui.config.environment "dev"
-                                                     "re_frame.trace.trace_enabled_QMARK_" true}
-                                   :external-config {:devtools/config {:features-to-install :all}}}}
+                        :compiler {:main "name-bazaar.server.core"
+                                   :output-to "server/name-bazaar.js",
+                                   :output-dir "server",
+                                   :target :nodejs,
+                                   :optimizations :simple,
+                                   :source-map "server/name-bazaar.js.map"
+                                   :closure-defines {goog.DEBUG false}
+                                   :pretty-print false}}
 
-                       ;; Development on server-side with testnet
+                       ;; server testnet
                        {:id "dev-server"
                         :source-paths ["src" "dev"]
                         :figwheel {:on-jsload "name-bazaar.server.dev/on-jsload"}
@@ -183,48 +176,41 @@
                                    :closure-defines {goog.DEBUG true}
                                    :source-map true}}
 
-                       ;; Production on server-side with mainnet
-                       {:id "server"
-                        :source-paths ["src"]
-                        :compiler {:main "name-bazaar.server.core"
-                                   :output-to "server/name-bazaar.js",
-                                   :output-dir "server",
+                       ;; server testing testnet
+                       {:id "server-tests"
+                        :source-paths ["src" "test"]
+                        :figwheel true
+                        :compiler {:main "server.run-tests"
+                                   :output-to "server-tests/server-tests.js",
+                                   :output-dir "server-tests",
                                    :target :nodejs,
-                                   :optimizations :simple,
-                                   :source-map "server/name-bazaar.js.map"
-                                   :closure-defines {goog.DEBUG false}
-                                   :pretty-print false}}
+                                   :optimizations :none,
+                                   :verbose false
+                                   :source-map true}}
 
-                       {:id "qa-min"
+                       ;; UI testnet
+                       {:id "dev-ui"
                         :source-paths ["src"]
+                        :figwheel {:on-jsload "name-bazaar.ui.core/mount-root"}
                         :compiler {:main "name-bazaar.ui.core"
                                    :output-to "resources/public/js/compiled/app.js"
-                                   :output-dir "resources/public/js/compiled"
-                                   :optimizations :advanced
-                                   :closure-defines {name-bazaar.ui.config.environment "qa"}
-                                   :source-map "resources/public/js/compiled/app.js.map"
-                                   :pretty-print false
-                                   :pseudo-names false}}
+                                   :output-dir "resources/public/js/compiled/out"
+                                   :asset-path "/js/compiled/out"
+                                   :source-map-timestamp true
+                                   :optimizations :none
+                                   :preloads [print.foo.preloads.devtools
+                                              day8.re-frame-10x.preload]
+                                   :closure-defines {"re_frame.trace.trace_enabled_QMARK_" true}
+                                   :external-config {:devtools/config {:features-to-install :all}}}}
 
-                       ;; Production on client-side with mainnet
+                       ;; UI mainnet
                        {:id "min"
                         :source-paths ["src"]
                         :compiler {:main "name-bazaar.ui.core"
                                    :output-to "resources/public/js/compiled/app.js"
+                                   :output-dir "resources/public/js/compiled"
                                    :optimizations :advanced
-                                   :closure-defines {name-bazaar.ui.config.environment "prod"}
+                                   :source-map "resources/public/js/compiled/app.js.map"
                                    :pretty-print false
-                                   :pseudo-names false}}
-
-                       ;; Testing on server-side
-                       {:id "server-tests"
-                        :source-paths ["src" "test"]
-                        :figwheel true
-                        :compiler {:main "server.run-tests"
-                                   :output-to "server-tests/server-tests.js",
-                                   :output-dir "server-tests",
-                                   :target :nodejs,
-                                   :optimizations :none,
-                                   :verbose false
-                                   :source-map true}}]})
+                                   :pseudo-names false}}]})
 

src/district/server/smart_contracts.cljs

@@ -2,7 +2,7 @@
   (:require [cljs-web3-next.core :as web3-core]
             [cljs-web3-next.eth :as web3-eth]
             [cljs-web3-next.helpers :as web3-helpers]
-            [cljs.core.async :refer [<! timeout] :as async]
+            [cljs.core.async :refer [<! >! timeout] :as async]
             [cljs.core.async.impl.protocols]
             [cljs.nodejs :as nodejs]
             [clojure.set :as clojure-set]
@@ -261,59 +261,86 @@
                              identity)]
     (transform-fn (remove-log-indexes sorted-logs))))
 
-(defn chunk->logs [transform-fn from-block skip-log-indexes events ignore-forward? [from to] ch-output]
-  "async/>! to ch-output for chunk [from to]: final sorted, skipped and transformed logs as async/ch."
-  (let [sort-and-skip-logs' (partial sort-and-skip-logs transform-fn from-block skip-log-indexes)]
-    (async/go
-      (->> (for [[k [contract event]] events
-                 :let [contract-instance (instance-from-arg contract {:ignore-forward? ignore-forward?})
-                       ch-logs (async/chan 1)]]
-             (do
-               (web3-eth/get-past-events contract-instance
-                                         event
-                                         {:from-block from
-                                          :to-block to}
-                                         (fn [error events]
-                                           (let [logs (->> events
-                                                           web3-helpers/js->cljkk
-                                                           (map (partial enrich-event-log contract contract-instance)))]
-                                             (async/put! ch-logs (or logs [(with-meta {:err error} {:error? true})]))
-                                             (async/close! ch-logs))))
-               ch-logs))
-           (async/merge)
-           (async/reduce into [])
-           (async/<!)
-           (sort-and-skip-logs')
-           (async/>! ch-output))
-      (async/close! ch-output))))
+(defn chunk->logs [transform-fn from-block skip-log-indexes events ignore-forward? re-try [from to] ch-output]
+  ">! to ch-output for chunk [from to]: final sorted, skipped and transformed logs as async/ch."
+  (let [sort-and-skip-logs' (partial sort-and-skip-logs transform-fn from-block skip-log-indexes)
+        ch-logs (async/chan 1)
+        event->logs (fn [[k [contract event]] ch-logs-output]
+                      (let [contract-instance (instance-from-arg contract {:ignore-forward? ignore-forward?})]
+                        (web3-eth/get-past-events contract-instance
+                                                  event
+                                                  {:from-block from
+                                                   :to-block to}
+                                                  (fn [error events]
+                                                    (let [logs (map (partial enrich-event-log contract contract-instance)
+                                                                    (web3-helpers/js->cljkk events))]
+                                                      (async/put! ch-logs-output (or logs [(with-meta {:err error} {:error? true})])))))))
+        re-try-event->logs (fn [event ch-output]
+                             (let [buf (async/chan 1)]
+                               (go-loop [re-try-iteration re-try]
+                                 (event->logs event buf)
+                                 (let [log (<! buf)]
+                                   (if (and (:error? (meta log))
+                                            (< 0 re-try-iteration))
+                                     (do (<! (async/timeout 1000))
+                                         (recur (dec re-try-iteration)))
+                                     (>! ch-output log))))))]
+    (go-loop [all-logs []
+              [event & rest-events] events]
+      (if event
+        (do
+          (re-try-event->logs event ch-logs)
+          (recur (into all-logs (<! ch-logs))
+                 rest-events))
+        (do
+          (async/close! ch-logs)
+          (>! ch-output (sort-and-skip-logs' all-logs))
+          (async/close! ch-output))))))
 
 (defn replay-past-events-in-order
   "Replay all past events in order.
   :from-block specifies the first block number events should be dispatched.
   :skip-log-indexes, a set of tuples like [tx log-index] for the :from-block block that should be skipped."
-  [events callback {:keys [from-block skip-log-indexes to-block block-step
-                           ignore-forward? crash-on-event-fail?
+  [events callback {:keys [from-block skip-log-indexes to-block block-step chunks-parallelism
+                           ignore-forward? crash-on-event-fail? re-try
                            transform-fn on-chunk on-finish]
-                    :or {transform-fn identity
+                    :or {chunks-parallelism 1
+                         re-try 3
+                         transform-fn identity
                          on-chunk :do-nothing
                          on-finish :do-nothing}
                     :as opts}]
 
-  (when (and skip-log-indexes (not from-block ))
+  (when (and skip-log-indexes (not from-block))
     (throw (js/Error. "replay-past-events-in-order: Can't specify skip-log-indexes without specifying :from-block")))
 
   (let [ch-chunks-to-process (async/to-chan! (all-chunks from-block to-block block-step))
         ch-final-logs (async/chan 1)
-        chunk->logs' (partial chunk->logs transform-fn from-block skip-log-indexes events ignore-forward?)]
-    (async/pipeline-async 10 ch-final-logs chunk->logs' ch-chunks-to-process)
-    (go-loop [chunk-logs (async/<! ch-final-logs)]
-      (if chunk-logs
+        chunk->logs' (partial chunk->logs transform-fn from-block skip-log-indexes events ignore-forward? re-try)
+        chs-await-for-workers (for [n (range chunks-parallelism)]
+                                (async/chan 1))
+        workers (dotimes [n chunks-parallelism]
+                  (go-loop []
+                    (try
+                      (if-let [chunk (<! ch-chunks-to-process)]
+                        (let [ch-wait (async/chan 1)]
+                          (chunk->logs' chunk ch-wait)
+                          (>! ch-final-logs (<! ch-wait))
+                          (recur))
+                        (async/close! (nth chs-await-for-workers n)))
+                      (catch ExceptionInfo e
+                        (log/error e)))))
+        all-workers-finished (go-loop []
+                               (<! (async/merge chs-await-for-workers))
+                               (async/close! ch-final-logs))]
+    (go-loop []
+      (if-let [chunk-logs (<! ch-final-logs)]
         (do
           (when (fn? callback)
             (doseq [log chunk-logs]
               (doseq [res (try
-                            (if-let [?error (:error? (meta log))]
-                              (callback ?error nil)
+                            (if (:error? (meta log))
+                              (callback log nil)
                               (callback nil log))
                             (catch js/Error e
                               (when crash-on-event-fail?
@@ -327,7 +354,7 @@
                   (async-helpers/promise? res)
                   (<! (async-helpers/promise->chan res))))))
           (on-chunk chunk-logs)
-          (recur (async/<! ch-final-logs)))
+          (recur))
 
         (on-finish)))))
 

src/name_bazaar/server/core.cljs

@@ -43,6 +43,7 @@
                                                    :offering-registry/offering-changed [:offering-registry :on-offering-changed]
                                                    :registrar/transfer [:eth-registrar :Transfer]}
                                           :from-block "latest"
+                                          :chunks-parallelism 10
                                           :block-step 1000}
                             :ui {:reveal-period {:hours 48}
                                  :etherscan-url "https://etherscan.io"