HYRAX-2012: Add -fPIC to the build flags for proj

.travis.yml

@@ -73,7 +73,6 @@ jobs:
     name: "rocky8 dynamic"
     script:
       - export ARTIFACT="rocky8"
-      - export EXPECTED_DEPS_COUNT=11
       - echo "# $ARTIFACT - Expecting to build and install $EXPECTED_DEPS_COUNT dependencies." >&2
       - export extra_targets=""
       - echo "# $ARTIFACT - extra_targets '$extra_targets'" >&2
@@ -101,13 +100,12 @@ jobs:
     name: "rocky9 dynamic"
     script:
       - export ARTIFACT="rocky9"
-      - export EXPECTED_DEPS_COUNT=11
       - echo "# $ARTIFACT - Expecting to build and install $EXPECTED_DEPS_COUNT dependencies." >&2
       - export extra_targets=""
       - echo "# $ARTIFACT - extra_targets '$extra_targets'" >&2
       - install_dir="$HOME/$ARTIFACT/install"
       - echo "# $ARTIFACT - install_dir '$install_dir'" >&2
-      - mkdir -vp $"install_dir"
+      - mkdir -vp "$install_dir"
       - docker run 
         --env prefix=/root/install 
         --env BUILD_NUMBER="$TRAVIS_BUILD_NUMBER"

GDAL-notes.txt

@@ -1,4 +1,53 @@
 
+# Notes about the GDAL builds 
+
+## A Makefile sniplet for GDAL 3.9.3 
+
+# The 3.9.3 cmake build of GDAL did not work on OSX 15.7.1, but did
+# work on a previous version of 15.x jhrg 10/24/25
+# GDAL
+#gdal_src=$(src)/$(gdal)
+#gdal_prefix=$(prefix)/deps
+#
+#$(gdal_src)-stamp:
+#	tar -xzf downloads/$(gdal_dist) -C $(src)
+#	echo timestamp > $(gdal_src)-stamp
+#
+## Set build options here (a few) and (most) in gdal-config.cmake.
+## jhrg 11/30/22
+#gdal-configure-stamp: $(gdal_src)-stamp
+#	mkdir -p $(gdal_src)/build
+#	(cd $(gdal_src)/build \
+#	 && cmake \
+#	 -DPROJ_INCLUDE_DIR=$(proj_prefix)/include \
+#	 -DPROJ_LIBRARY_RELEASE=$(proj_prefix)/lib/libproj.a \
+#	 -DCMAKE_INSTALL_PREFIX:PATH=$(prefix)/deps \
+#	 -DCMAKE_C_FLAGS="-fPIC -O2" \
+#	 $(CMAKE_FLAGS) \
+#	 -C ../../../gdal-config.cmake ..)
+#	echo timestamp > gdal-configure-stamp
+#
+#gdal-compile-stamp: gdal-configure-stamp
+#	(cd $(gdal_src)/build && $(MAKE) $(MFLAGS))
+#	echo timestamp > gdal-compile-stamp
+#
+#gdal-install-stamp: gdal-compile-stamp
+#	(cd $(gdal_src)/build && $(MAKE) $(MFLAGS) -j1 install)
+#	echo timestamp > gdal-install-stamp
+#
+#gdal-clean:
+#	-rm gdal-*-stamp
+#	-(cd  $(gdal_src)/build && $(MAKE) $(MFLAGS) clean)
+#
+#gdal-really-clean: gdal-clean
+#	-rm $(gdal_src)-stamp
+#	-rm -rf $(gdal_src)
+#
+#.PHONY: gdal
+#gdal: gdal-install-stamp
+
+## Info about a cmake-based build 
+
 cmake -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF
 -DOGR_BUILD_OPTIONAL_DRIVERS=OFF
 -DGDAL_USE_XERCESC=OFF

Makefile

@@ -330,49 +330,35 @@ $(gdal_src)-stamp:
 
 gdal-configure-stamp: $(gdal_src)-stamp
 	(cd $(gdal_src) && \
-	export CPPFLAGS="$(CPPFLAGS) -I$(proj_prefix)/include -I/opt/homebrew/Cellar/libgeotiff/1.7.4/include";\
+	export CPPFLAGS="$(CPPFLAGS) -I$(proj_prefix)/include -I/opt/homebrew/Cellar/libgeotiff/1.7.4/include"; \
 	export LDFLAGS="$$LDFLAGS -L$$prefix/deps/lib -Wl,-rpath -Wl,$$prefix/deps/lib \
 	    -L$$prefix/deps/proj/lib -Wl,-rpath -Wl,$$prefix/deps/proj/lib -lpthread -lm "; \
-	export proj_libdir="$(proj_prefix)/lib64" ; \
-	export deps_libdir="$(prefix)/deps/lib64"; \
-	if ! test -d "$$proj_libdir"; then proj_libdir="$(proj_prefix)/lib"; \
-		if [[ "$$OSTYPE" == "darwin"* ]]; then \
-			echo "# Building on OSX, LDFLAGS unchanged"; \
-		else \
-			echo "# Not building on OSX; updating LDFLAGS"; \
-			export LDFLAGS="$$LDFLAGS -L$$proj_libdir -lproj"; \
-		fi; \
-	fi ; \
-	if ! test -d "$$deps_libdir"; then export deps_libdir="$(prefix)/deps/lib"; fi; \
+	export proj_libdir="$(proj_prefix)/lib"; \
+	export deps_libdir="$(prefix)/deps/lib"; \
+	if test -d "$(proj_prefix)/lib64"; then \
+		export proj_libdir="$$proj_libdir $(proj_prefix)/lib64"; \
+	fi; \
+	if test -d "$(prefix)/deps/lib64"; then \
+		export deps_libdir="$$deps_libdir $(prefix)/deps/lib64"; \
+	fi; \
 	if [[ "$$OSTYPE" == "darwin"* ]]; then \
 		export PKG_CONFIG_PATH=$(prefix)/deps/lib/pkgconfig; \
 	else \
-		export PKG_CONFIG_PATH="$$proj_libdir/pkgconfig:$$deps_libdir/pkgconfig"; \
+		for dir in $$proj_libdir $$deps_libdir; do \
+			if test -d "$$dir"; then \
+				export LDFLAGS="$$LDFLAGS -L$$dir -Wl,-rpath -Wl,$$dir"; \
+			fi; \
+			if test -d "$$dir/pkgconfig"; then \
+				if test -n "$$PKG_CONFIG_PATH"; then \
+					export PKG_CONFIG_PATH="$$PKG_CONFIG_PATH:$$dir/pkgconfig"; \
+				else \
+					export PKG_CONFIG_PATH="$$dir/pkgconfig"; \
+				fi; \
+			fi; \
+		done; \
+		export LDFLAGS="$$LDFLAGS -lproj"; \
 	fi; \
-	echo "###################################################################"; \
-	echo "#     proj_libdir: '$$proj_libdir'"; \
-	echo "#     deps_libdir: '$$deps_libdir'"; \
-	echo "# PKG_CONFIG_PATH: '$$PKG_CONFIG_PATH'"; \
-	echo "#        CPPFLAGS: '$$CPPFLAGS'"; \
-	echo "#         LDFLAGS: '$$LDFLAGS'"; \
-	echo "#          OSTYPE: '$$OSTYPE'"; \
-	echo "#"; \
-	pkg-config --list-all | awk '{print "## "$$0; }' - ; \
-	echo "#"; \
-	echo "# ls -l $$proj_libdir "; \
-	ls -l "$$proj_libdir" ; \
-	echo "#"; \
-	echo "# ls -l $$proj_libdir/pkgconfig: "; \
-	ls -l $$proj_libdir/pkgconfig; \
-	echo "#"; \
-	echo "# awk '{print "## "$$0;}' $$proj_libdir/pkgconfig/proj.pc: "; \
-	awk '{print "## "$$0;}' $$proj_libdir/pkgconfig/proj.pc; \
-	echo "#"; \
-	echo "# pkg-config --exists proj"; \
-	pkg-config --exists proj; \
-	if test $$? -eq 0 ; then echo "# pkg-config FOUND proj"; else echo "# pkg-config FAILED to find proj"; fi ; \
-	echo "#"; \
-	echo "###################################################################"; \
+	bash ../../travis/debug-proj-pkg-config.sh; \
 	./configure $(CONFIGURE_FLAGS) --prefix=$(gdal_prefix) --with-pic \
 	--with-openjpeg --without-jasper --disable-all-optional-drivers \
 	--enable-driver-grib $(LIBPNG) --with-proj=$(proj_prefix) \
@@ -420,7 +406,7 @@ $(proj_src)-stamp:
 proj-configure-stamp: $(proj_src)-stamp
 	mkdir -p $(proj_src)/build
 	(cd $(proj_src)/build \
-	 && cmake -DCMAKE_INSTALL_PREFIX=$(proj_prefix) $(CMAKE_FLAGS) \
+	 && cmake -DCMAKE_INSTALL_PREFIX=$(proj_prefix) -DCMAKE_C_FLAGS="-fPIC -O2" $(CMAKE_FLAGS) \
         -DENABLE_TIFF:bool=OFF -DCMAKE_PREFIX_PATH=$(prefix)/deps ..)
 	echo timestamp > proj-configure-stamp
 
@@ -444,49 +430,6 @@ proj-really-clean: proj-clean
 .PHONY: proj
 proj: proj-install-stamp
 
-# The 3.9.3 cmake build of GDAL did not work on OSX 15.7.1, but did
-# work on a previous version of 15.x jhrg 10/24/25
-# GDAL
-#gdal_src=$(src)/$(gdal)
-#gdal_prefix=$(prefix)/deps
-#
-#$(gdal_src)-stamp:
-#	tar -xzf downloads/$(gdal_dist) -C $(src)
-#	echo timestamp > $(gdal_src)-stamp
-#
-## Set build options here (a few) and (most) in gdal-config.cmake.
-## jhrg 11/30/22
-#gdal-configure-stamp: $(gdal_src)-stamp
-#	mkdir -p $(gdal_src)/build
-#	(cd $(gdal_src)/build \
-#	 && cmake \
-#	 -DPROJ_INCLUDE_DIR=$(proj_prefix)/include \
-#	 -DPROJ_LIBRARY_RELEASE=$(proj_prefix)/lib/libproj.a \
-#	 -DCMAKE_INSTALL_PREFIX:PATH=$(prefix)/deps \
-#	 -DCMAKE_C_FLAGS="-fPIC -O2" \
-#	 $(CMAKE_FLAGS) \
-#	 -C ../../../gdal-config.cmake ..)
-#	echo timestamp > gdal-configure-stamp
-#
-#gdal-compile-stamp: gdal-configure-stamp
-#	(cd $(gdal_src)/build && $(MAKE) $(MFLAGS))
-#	echo timestamp > gdal-compile-stamp
-#
-#gdal-install-stamp: gdal-compile-stamp
-#	(cd $(gdal_src)/build && $(MAKE) $(MFLAGS) -j1 install)
-#	echo timestamp > gdal-install-stamp
-#
-#gdal-clean:
-#	-rm gdal-*-stamp
-#	-(cd  $(gdal_src)/build && $(MAKE) $(MFLAGS) clean)
-#
-#gdal-really-clean: gdal-clean
-#	-rm $(gdal_src)-stamp
-#	-rm -rf $(gdal_src)
-#
-#.PHONY: gdal
-#gdal: gdal-install-stamp
-
 # Gridfields 
 gridfields_src=$(src)/$(gridfields)
 gridfields_prefix=$(prefix)/deps

TRAVIS_PERFORMANCE_PLAN.md

@@ -0,0 +1,71 @@
+# Travis Performance Plan
+
+## Goals
+
+Improve the performance of [`.travis.yml`](/Users/jimg/src/opendap/hyrax/hyrax-dependencies/.travis.yml) and identify ways to simplify it without changing the end result.
+
+## Plan
+
+1. Establish a baseline and confirm where time is going.
+
+   Measure one or two recent runs by job type and break the runtime into `apt`, Docker startup, `dnf update`, dependency build, packaging, and deploy. This matters because the biggest likely cost in the current setup is not Travis YAML parsing, but repeated dependency rebuilds and the `dnf -y update` inside [travis/build-for-rocky8.sh](/Users/jimg/src/opendap/hyrax/hyrax-dependencies/travis/build-for-rocky8.sh#L57) and [travis/build-for-rocky9.sh](/Users/jimg/src/opendap/hyrax/hyrax-dependencies/travis/build-for-rocky9.sh#L58).
+
+2. Add Travis cache for downloaded sources and build state.
+
+   The Makefile is already stamp-driven, which is a good fit for caching. The first cache pass should preserve:
+
+   - `downloads/`
+   - `src/`
+   - repo-root `*-stamp` files
+   - job-specific install trees such as `$HOME/install`, `$HOME/rocky8/install`, and `$HOME/rocky9/install`
+
+   That gives Travis a chance to skip re-downloading, re-unpacking, re-configuring, and re-installing unchanged dependencies on later runs.
+
+3. Split caches by platform and add explicit cache-bust controls.
+
+   Do not share one cache across Ubuntu, Rocky 8, and Rocky 9. Their binaries and toolchains are different. Use separate cache namespaces per job, and add a simple manual cache version variable so the cache can be invalidated cleanly when [Makefile](/Users/jimg/src/opendap/hyrax/hyrax-dependencies/Makefile), dependency versions, or the Rocky build scripts change.
+
+4. Start with safe caching, then optionally expand to installed artifacts.
+
+   Phase 1 should cache only source downloads, extracted source trees, and stamp files. That is low-risk and should already remove a lot of rebuild work.
+
+   Phase 2 can cache the installed prefixes too, but only after validating that reused installs remain correct when compiler flags or dependency versions change. This is where a cache version key becomes important.
+
+5. Remove or reduce the unconditional `dnf update` in the Rocky jobs.
+
+   Both Rocky scripts do a full `dnf -y update` on every run. That is usually expensive and tends to defeat repeatability as well as caching. The better plan is:
+
+   - build or refresh the Docker images ahead of time with the needed packages baked in, then stop doing the full update in Travis
+   - or make the update conditional so it only runs when intentionally refreshing the builder image
+
+   This is likely one of the highest-impact improvements.
+
+6. Decide whether Docker image pull and update behavior should be stabilized.
+
+   If the Travis host is always pulling a moving `:latest` image, builds can slow down and become less reproducible. Pinning [`.travis.yml`](/Users/jimg/src/opendap/hyrax/hyrax-dependencies/.travis.yml#L82) and [`.travis.yml`](/Users/jimg/src/opendap/hyrax/hyrax-dependencies/.travis.yml#L109) to versioned builder tags would help both performance predictability and cache usefulness.
+
+7. Make packaging and deploy work happen only after a successful cache-aware build check.
+
+   Each job currently creates tarballs after `make`. Once caching is in place, keep the packaging step but make sure the build verification still proves the cache was valid, for example with `make list-built` and [travis/check-installed](/Users/jimg/src/opendap/hyrax/hyrax-dependencies/travis/check-installed#L1). This preserves the end result while reducing unnecessary compilation.
+
+8. Add a short validation phase for cache correctness.
+
+   Run the jobs twice in a row on the same branch and compare logs. The second run should show most dependency targets being skipped because the stamp files and install trees are already present. Then test one intentional dependency version change to confirm the cache-bust process works.
+
+## Separate Simplification Step
+
+9. Simplify `.travis.yml` without changing outputs.
+
+   The three jobs in [`.travis.yml`](/Users/jimg/src/opendap/hyrax/hyrax-dependencies/.travis.yml#L53) repeat a lot of setup, packaging, and logging. A clean simplification pass would:
+
+   - use shared YAML anchors or a small wrapper script for the repeated tarball and packaging logic
+   - parameterize the Rocky jobs by image name, artifact name, and helper script instead of duplicating nearly identical blocks
+   - remove dead or unused structure such as the `never` stage if it is no longer serving a purpose
+   - fix minor cleanup issues such as the likely typo `mkdir -vp $"install_dir"` in [`.travis.yml`](/Users/jimg/src/opendap/hyrax/hyrax-dependencies/.travis.yml#L107), which should be `"$install_dir"`
+
+## Recommended Order
+
+1. Cache `downloads/`, `src/`, and stamp files.
+2. Remove `dnf -y update` from per-build execution by baking it into the Rocky images.
+3. Add optional caching for installed prefixes.
+4. Simplify `.travis.yml` by deduplicating job logic.

travis/debug-proj-pkg-config.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+echo "###################################################################"
+echo "#     proj_libdir: '$proj_libdir'"
+echo "#     deps_libdir: '$deps_libdir'"
+echo "# PKG_CONFIG_PATH: '$PKG_CONFIG_PATH'"
+echo "#        CPPFLAGS: '$CPPFLAGS'"
+echo "#         LDFLAGS: '$LDFLAGS'"
+echo "#          OSTYPE: '$OSTYPE'"
+echo "#"
+pkg-config --list-all | awk '{print "## "$0; }' -
+for dir in $proj_libdir; do
+    echo "#"
+    echo "# ls -l $dir "
+    ls -l "$dir"
+    if test -d "$dir/pkgconfig"; then
+        echo "#"
+        echo "# ls -l $dir/pkgconfig: "
+        ls -l "$dir/pkgconfig"
+    fi
+    if test -f "$dir/pkgconfig/proj.pc"; then
+        echo "#"
+        echo "# awk '{print \"## \"\$0;}' $dir/pkgconfig/proj.pc: "
+        awk '{print "## "$0;}' "$dir/pkgconfig/proj.pc"
+    fi
+done
+echo "#"
+echo "# pkg-config --exists proj"
+pkg-config --exists proj
+if test $? -eq 0; then echo "# pkg-config FOUND proj"; else echo "# pkg-config FAILED to find proj"; fi
+echo "#"
+echo "###################################################################"