diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 0000000..4edede3 --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,75 @@ +name: docker build + +on: + schedule: + - cron: "0 10 * * *" + push: + branches: + - "**" + tags: + - "v*.*.*" + pull_request: + branches: + - "master" + +permissions: + contents: read + packages: write + +jobs: + docker: + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: Set variables useful for later + id: useful_vars + run: |- + echo "::set-output name=timestamp::$(date +%s)" + echo "::set-output name=short_sha::${GITHUB_SHA::8}" + - name: Checkout + uses: actions/checkout@v3 + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,prefix=,format=long,event=tag + type=sha + type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} + type=raw,value=${{ github.ref_name }}-${{ steps.useful_vars.outputs.short_sha }}-${{ steps.useful_vars.outputs.timestamp }},enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache,mode=max diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1a4568a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN apt-get update && \ + apt-get -y install gcc && \ + pip install --no-cache-dir -r requirements.txt && \ + rm -rf /var/lib/apt/lists/* +COPY . . +RUN python3 setup.py build_ext --inplace +COPY . . +ENTRYPOINT [ "python", "/app/mlat-server"] diff --git a/mlat/config.py b/mlat/config.py index 1eb2e2a..bb54c20 100644 --- a/mlat/config.py +++ b/mlat/config.py @@ -29,7 +29,7 @@ # # Please remember that this needs to be _the code that the server is running_. # -AGPL_SERVER_CODE_URL = "https://github.com/wiedehopf/mlat-server" +AGPL_SERVER_CODE_URL = "https://github.com/katlol/mlat-server" # minimum NUCp value to accept as a sync message MIN_NUC = 6 diff --git a/mlat/coordinator.py b/mlat/coordinator.py index 7d59205..65bd283 100644 --- a/mlat/coordinator.py +++ b/mlat/coordinator.py @@ -421,10 +421,6 @@ def _write_state(self): 'user': r.user, 'uid': r.uid, 'uuid': r.uuid, - 'coords': "{0:.6f},{1:.6f}".format(r.position_llh[0], r.position_llh[1]), - 'lat': r.position_llh[0], - 'lon': r.position_llh[1], - 'alt': r.position_llh[2], 'privacy': r.privacy, 'connection': r.connection_info, 'source_ip': r.connection.source_ip, @@ -482,24 +478,24 @@ def _write_state(self): cpu_time_us_per_sec = round((cpu_time - self.last_cpu_time) * (1e6 / 15)) self.last_cpu_time = cpu_time try: - with open('/run/node_exporter/mlat-server.prom', 'w', encoding='utf-8') as f: - out = '' - out += 'mlat_server_cpu_ppm ' + str(cpu_time_us_per_sec) + '\n' - out += 'mlat_server_receivers ' + str(len(self.receivers)) + '\n' - out += 'mlat_server_ac_mlat ' + str(ac_count_mlat) + '\n' - out += 'mlat_server_ac_sync ' + str(ac_count_sync) + '\n' - out += 'mlat_server_ac_total ' + str(len(self.tracker.aircraft)) + '\n' - out += 'mlat_server_outlier_ppm ' + "{0:.0f}".format(total_outlier_percent * 1000) + '\n' - out += 'mlat_server_sync_points ' + "{0:.0f}".format(self.stats_sync_points / self.main_interval) + '\n' - out += 'mlat_server_sync_msgs ' + "{0:.0f}".format(self.stats_sync_msgs / self.main_interval) + '\n' - out += 'mlat_server_mlat_msgs ' + "{0:.0f}".format(self.stats_mlat_msgs / self.main_interval) + '\n' - out += 'mlat_server_valid_groups ' + "{0:.0f}".format(self.stats_valid_groups / self.main_interval) + '\n' - out += 'mlat_server_normalize_called ' + "{0:.0f}".format(self.stats_normalize / self.main_interval) + '\n' - out += 'mlat_server_solve_attempt ' + "{0:.0f}".format(self.stats_solve_attempt / self.main_interval) + '\n' - out += 'mlat_server_solve_success ' + "{0:.0f}".format(self.stats_solve_success / self.main_interval) + '\n' - out += 'mlat_server_solve_used ' + "{0:.0f}".format(self.stats_solve_used / self.main_interval) + '\n' - - f.write(out) + out = [ + 'mlat_server_cpu_ppm ' + str(cpu_time_us_per_sec) + '\n', + 'mlat_server_receivers ' + str(len(self.receivers)) + '\n', + 'mlat_server_ac_mlat ' + str(ac_count_mlat) + '\n', + 'mlat_server_ac_sync ' + str(ac_count_sync) + '\n', + 'mlat_server_ac_total ' + str(len(self.tracker.aircraft)) + '\n', + 'mlat_server_outlier_ppm ' + "{0:.0f}".format(total_outlier_percent * 1000) + '\n', + 'mlat_server_sync_points ' + "{0:.0f}".format(self.stats_sync_points / self.main_interval) + '\n', + 'mlat_server_sync_msgs ' + "{0:.0f}".format(self.stats_sync_msgs / self.main_interval) + '\n', + 'mlat_server_mlat_msgs ' + "{0:.0f}".format(self.stats_mlat_msgs / self.main_interval) + '\n', + 'mlat_server_valid_groups ' + "{0:.0f}".format(self.stats_valid_groups / self.main_interval) + '\n', + 'mlat_server_normalize_called ' + "{0:.0f}".format(self.stats_normalize / self.main_interval) + '\n', + 'mlat_server_solve_attempt ' + "{0:.0f}".format(self.stats_solve_attempt / self.main_interval) + '\n', + 'mlat_server_solve_success ' + "{0:.0f}".format(self.stats_solve_success / self.main_interval) + '\n', + 'mlat_server_solve_used ' + "{0:.0f}".format(self.stats_solve_used / self.main_interval) + '\n', + ] + with open('/run/mlat-server/metrics', 'w', encoding='utf-8') as f: + f.writelines(out) except OSError: pass except: diff --git a/mlat/jsonclient.py b/mlat/jsonclient.py index 2a47097..8f85a89 100644 --- a/mlat/jsonclient.py +++ b/mlat/jsonclient.py @@ -340,6 +340,13 @@ async def process_handshake(self): if hs['version'] != 2 and hs['version'] != 3: raise ValueError('Unsupported version in handshake') + # If no compress key in hs, then the client is using an old version. see ya later! + if 'compress' not in hs.keys(): + raise ValueError('Unsupported: Missing compression') + # also, check against self._compression_methods + if not set(hs['compress']).issubset(set([c[0] for c in self._compression_methods])): + raise ValueError('Unsupported compression type, got: ' + str(hs)) + user = str(hs['user']) uuid = hs.get('uuid') @@ -375,6 +382,10 @@ async def process_handshake(self): if self.compress is None: raise ValueError('No mutually usable compression type') + if self.handle_messages is None: + print(hs) # FIXME temp + raise ValueError('No handle_messages method. Why??') + lat = float(hs['lat']) if lat < -90 or lat > 90: raise ValueError('invalid latitude, should be -90 .. 90') diff --git a/mlat/solver.py b/mlat/solver.py index d0870c7..58791b6 100644 --- a/mlat/solver.py +++ b/mlat/solver.py @@ -28,6 +28,7 @@ from mlat import geodesy, constants, profile from mlat import config +import numpy as np # The core of it all. Not very big, is it? # (Admittedly the entire least-squares solver is hidden within scipy..) @@ -36,7 +37,7 @@ def _residuals(x_guess, pseudorange_data, altitude, altitude_error): - """Return an array of residuals for a position guess at x_guess versus + """Return the sum of squared residuals for a position guess at x_guess versus actual measurements pseudorange_data and altitude.""" (*position_guess, offset) = x_guess @@ -53,7 +54,8 @@ def _residuals(x_guess, pseudorange_data, altitude, altitude_error): _, _, altitude_guess = geodesy.ecef2llh(position_guess) res.append((altitude - altitude_guess) / altitude_error) - return res + # return the sum of squared residuals + return np.sum(np.array(res)**2) @profile.trackcpu @@ -91,19 +93,19 @@ def solve(measurements, altitude, altitude_error, initial_guess): math.sqrt(variance) * constants.Cair) for receiver, timestamp, variance in measurements] x_guess = [initial_guess[0], initial_guess[1], initial_guess[2], 0.0] - x_est, cov_x, infodict, mesg, ler = scipy.optimize.leastsq( + res = scipy.optimize.minimize( _residuals, x_guess, args=(pseudorange_data, altitude, altitude_error), - full_output=True, - maxfev=config.SOLVER_MAXFEV) + method='BFGS', + options={'maxiter': config.SOLVER_MAXFEV}) - if ler in (1, 2, 3, 4): + if res.success: #glogger.info("solver success: {0} {1}".format(ler, mesg)) # Solver found a result. Validate that it makes # some sort of physical sense. - (*position_est, offset_est) = x_est + (*position_est, offset_est) = res.x if offset_est < 0 or offset_est > 500e3: #glogger.info("solver: bad offset: {0}".formaT(offset_est)) @@ -118,10 +120,10 @@ def solve(measurements, altitude, altitude_error, initial_guess): # #glogger.info("solver: bad range: {0}".format(d)) # return None - if cov_x is None: + if res.hess_inv is None: return position_est, None else: - return position_est, cov_x[0:3, 0:3] + return position_est, res.hess_inv[0:3, 0:3] else: # Solver failed diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..39ba290 --- /dev/null +++ b/requirements.in @@ -0,0 +1,7 @@ +numpy +scipy +pykalman +python-graph-core +uvloop +ujson +Cython diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7258bd7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile +# +cython==3.0.0 + # via -r requirements.in +numpy==1.25.1 + # via + # -r requirements.in + # scipy +pykalman==0.9.5 + # via -r requirements.in +python-graph-core==1.8.2 + # via -r requirements.in +scipy==1.11.1 + # via -r requirements.in +ujson==5.8.0 + # via -r requirements.in +uvloop==0.17.0 + # via -r requirements.in