minor edits

Author: atharvas

scratch/notebooks/collect_perf_commits.ipynb

@@ -2,7 +2,7 @@
   "cells": [
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": 7,
       "id": "13626f75",
       "metadata": {},
       "outputs": [
@@ -12,13 +12,6 @@
           "text": [
             "/mnt/sdd1/atharvas/formulacode/datasmith\n"
           ]
-        },
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "09:16:49 WARNING  simple_useragent.core: Falling back to historic user agent.\n"
-          ]
         }
       ],
       "source": [
@@ -35,7 +28,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 2,
+      "execution_count": 11,
       "id": "b4179f19",
       "metadata": {},
       "outputs": [],
@@ -49,12 +42,13 @@
         "    --limit-per-repo 2\n",
         "    --max-attempts 3\n",
         "    --max-steps 10\n",
-        "\"\"\".strip().replace(\"\\n\", \" \")"
+        "\"\"\".strip().replace(\"\\n\", \" \")\n",
+        "cr = ContextRegistry.load_from_file(Path(\"scratch/artifacts/pipeflush/context_registry.json\"))"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 3,
+      "execution_count": null,
       "id": "6624689c",
       "metadata": {},
       "outputs": [
@@ -230,15 +224,14 @@
         }
       ],
       "source": [
-        "cr = ContextRegistry.load_from_file(Path(\"scratch/artifacts/pipeflush/context_registry.json\"))\n",
         "commit_pth = Path(\"scratch/artifacts/pipeflush/commits_perfonly.parquet\")\n",
         "commit_df = pd.read_parquet(commit_pth)\n",
         "commit_df.head()"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 4,
+      "execution_count": 9,
       "id": "79905eb5",
       "metadata": {},
       "outputs": [
@@ -295,17 +288,17 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 5,
+      "execution_count": 12,
       "id": "567cdaa5",
       "metadata": {},
       "outputs": [
         {
           "name": "stderr",
           "output_type": "stream",
           "text": [
-            "09:17:16 INFO     datasmith.docker.context: Context registry saved to scratch/artifacts/pipeflush/chunk_0/context_registry.json\n",
-            "09:17:36 INFO     datasmith.docker.context: Context registry saved to scratch/artifacts/pipeflush/chunk_1/context_registry.json\n",
-            "09:17:58 INFO     datasmith.docker.context: Context registry saved to scratch/artifacts/pipeflush/chunk_2/context_registry.json\n"
+            "09:51:55 INFO     datasmith.docker.context: Context registry saved to scratch/artifacts/pipeflush/chunk_0/context_registry.json\n",
+            "09:51:59 INFO     datasmith.docker.context: Context registry saved to scratch/artifacts/pipeflush/chunk_1/context_registry.json\n",
+            "09:52:08 INFO     datasmith.docker.context: Context registry saved to scratch/artifacts/pipeflush/chunk_2/context_registry.json\n"
           ]
         }
       ],
@@ -338,7 +331,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 6,
+      "execution_count": 13,
       "id": "3fafdd1c",
       "metadata": {},
       "outputs": [
@@ -390,11 +383,31 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 20,
       "id": "f5133158",
       "metadata": {},
-      "outputs": [],
-      "source": []
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "array([47.51953125, 47.51785714, 47.52559055])"
+            ]
+          },
+          "execution_count": 20,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "import numpy as np\n",
+        "\n",
+        "lens = np.array([len(d) for d in [df1, df2, df3]])\n",
+        "ratios = np.array(ratios)\n",
+        "\n",
+        "l2r = lens * 15 / ratios  # min\n",
+        "l2r_hrs = l2r / 60\n",
+        "l2r_hrs"
+      ]
     }
   ],
   "metadata": {

scratch/scripts/update_context_registry.py

@@ -9,13 +9,15 @@ def update_cr_entrypoint(cr: ContextRegistry):
     for k, v in cr.registry.items():
         new_v = deepcopy(v)
         new_v.entrypoint_data = DockerContext().entrypoint_data
+        new_v.dockerfile_data = DockerContext().dockerfile_data
+        new_v.env_building_data = DockerContext().env_building_data
         new_reg[k] = new_v
 
     cr.registry = new_reg
     return cr
 
 
 if __name__ == "__main__":
-    cr = ContextRegistry.load_from_file(Path("scratch/context_registry.json"))
+    cr = ContextRegistry.load_from_file(Path("scratch/artifacts/pipeflush/tiny/context_registry.json"))
     new_cr = update_cr_entrypoint(cr)
-    new_cr.save_to_file(Path("scratch/context_registry_updated.json"))
+    new_cr.save_to_file(Path("scratch/artifacts/pipeflush/context_registry.json"))

src/datasmith/docker/Dockerfile

@@ -7,7 +7,7 @@ ARG COMMIT_SHA
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        curl git build-essential jq cmake ninja-build && \
+        jq cmake ninja-build && \
     rm -rf /var/lib/apt/lists/*
 
 RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest \
@@ -22,7 +22,7 @@ ENV MAMBA_ROOT_PREFIX=/opt/conda \
 
 RUN micromamba install -y -p $MAMBA_ROOT_PREFIX -c conda-forge \
         python=3.10 \
-        git asv pyperf mamba conda libmambapy jq && \
+        git asv pyperf mamba conda libmambapy && \
     micromamba clean --all --yes
 
 RUN mkdir -p /workspace /output