Skip to content

Commit 5213347

Browse files
rustyconoverclaude
andcommitted
Add Polars functions to example worker and fix scalar finalization
- Add all Polars scalar functions to ExampleWorker for vgi-client testing - Fix scalar function worker to send FINISHED status after finalize (was hanging because client expected a response) - Add position parameter to ConstParam for PolarsScalarFunction support - Update PolarsMultiplyFunction to use proper ConstParam declaration - Extract ConstParam from _polars_params in metadata extraction Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 3cd16ac commit 5213347

5 files changed

Lines changed: 67 additions & 22 deletions

File tree

vgi/arguments.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1588,6 +1588,8 @@ class ConstParam:
15881588
doc: Documentation string describing this parameter.
15891589
arrow_type: Optional explicit Arrow type. If not provided, type is
15901590
inferred from the Annotated first argument.
1591+
position: Position in the argument list (required for PolarsScalarFunction,
1592+
optional for ScalarFunction where position is inferred from signature).
15911593
15921594
Example:
15931595
class FormatNumber(ScalarFunction):
@@ -1600,10 +1602,20 @@ def compute(
16001602
fmt = f"%.{precision}f"
16011603
return pa.array([fmt % v for v in value.to_pylist()])
16021604
1605+
Example for PolarsScalarFunction:
1606+
class Multiply(PolarsScalarFunction):
1607+
value: Annotated[pl.Float64, Param(position=0, doc="Value to multiply")]
1608+
factor: Annotated[float, ConstParam("Factor", position=0)]
1609+
1610+
def compute_polars(self) -> pl.Expr:
1611+
return pl.col("value") * self.factor
1612+
16031613
"""
16041614

16051615
doc: str = ""
16061616
arrow_type: pa.DataType | type | None = None
1617+
# Position in the argument list (for PolarsScalarFunction class-level attributes)
1618+
position: int | None = None
16071619

16081620

16091621
@dataclass(frozen=True, slots=True)

vgi/examples/scalar_polars.py

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ def compute_polars(self) -> pl.Expr:
6262
import polars as pl
6363
import pyarrow.types as pat
6464

65-
from vgi.arguments import Param
65+
from vgi.arguments import ConstParam, Param
6666
from vgi.metadata import FunctionExample
6767
from vgi.scalar_function_polars import AnyPolars, PolarsScalarFunction
6868

@@ -253,17 +253,15 @@ class PolarsMultiplyFunction(PolarsScalarFunction):
253253
"""Multiply a numeric column by a constant factor using Polars.
254254
255255
Demonstrates combining a column parameter with a constant (scalar) argument.
256-
The constant factor is passed via SQL and accessed through
257-
``self.invocation.arguments.positional``.
256+
Use ``ConstParam`` to declare constant arguments that appear in function
257+
signatures and metadata.
258258
259259
This pattern is useful when you need a user-specified value that doesn't
260260
come from a table column (e.g., scaling factor, threshold, limit).
261261
262262
Attributes:
263-
value: Numeric column to multiply (position 0).
264-
265-
Properties:
266-
factor: The constant multiplication factor from arguments.
263+
value: Numeric column to multiply (position 0 in input batch).
264+
factor: Constant multiplication factor (position 0 in function arguments).
267265
268266
SQL Usage:
269267
``SELECT polars_multiply(price, 1.1) FROM products``
@@ -272,9 +270,15 @@ class PolarsMultiplyFunction(PolarsScalarFunction):
272270
>>> input: value=[10.0, 20.0, 30.0], factor=2.0
273271
>>> output: [20.0, 40.0, 60.0]
274272
273+
Note:
274+
The ``factor`` is a constant argument passed from SQL, not a table column.
275+
It's declared with ``ConstParam(position=0)`` to indicate it's the first
276+
positional argument in the function call.
277+
275278
"""
276279

277280
value: Annotated[pl.Float64, Param(position=0, doc="Value to multiply")]
281+
factor: Annotated[float, ConstParam("Multiplication factor", position=0)]
278282

279283
class Meta:
280284
"""Function metadata."""
@@ -290,13 +294,13 @@ class Meta:
290294
]
291295

292296
@property
293-
def factor(self) -> float:
297+
def _factor(self) -> float:
294298
"""Get the multiplication factor from arguments."""
295299
return self.invocation.arguments.positional[0].as_py() # type: ignore
296300

297301
def compute_polars(self) -> pl.Expr:
298302
"""Multiply the values by the constant factor."""
299-
return pl.col("value") * self.factor
303+
return pl.col("value") * self._factor
300304

301305

302306
# =============================================================================

vgi/examples/worker.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,15 @@
2929
SumValuesFunction,
3030
UpperCaseFunction,
3131
)
32+
from vgi.examples.scalar_polars import (
33+
PolarsAddValuesFunction,
34+
PolarsDoubleFunction,
35+
PolarsMultiplyFunction,
36+
PolarsNormalizeFunction,
37+
PolarsStringLengthFunction,
38+
PolarsSumValuesFunction,
39+
PolarsUpperCaseFunction,
40+
)
3241
from vgi.examples.table import (
3342
ConstantColumnsFunction,
3443
DoubleSequenceFunction,
@@ -103,6 +112,14 @@ class Settings:
103112
RandomIntFunction,
104113
SumValuesFunction,
105114
UpperCaseFunction,
115+
# PolarsScalarFunction - Polars-based scalar functions
116+
PolarsAddValuesFunction,
117+
PolarsDoubleFunction,
118+
PolarsMultiplyFunction,
119+
PolarsNormalizeFunction,
120+
PolarsStringLengthFunction,
121+
PolarsSumValuesFunction,
122+
PolarsUpperCaseFunction,
106123
]
107124

108125

vgi/metadata.py

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -480,30 +480,30 @@ def extract_parameters(
480480
parameters: list[ParameterInfo] = []
481481
seen_names: set[str] = set()
482482

483-
# Check for _polars_params (PolarsScalarFunction subclasses)
484-
# These store PolarsParamInfo objects with Polars types
483+
# Note: _polars_params (PolarsScalarFunction subclasses) with is_const=False
484+
# are NOT extracted as function parameters. They're column position bindings
485+
# that describe which columns from the input batch map to expression variables.
486+
# However, ConstParam entries (is_const=True) ARE function arguments and must
487+
# be extracted for the function signature.
485488
polars_params = getattr(cls, "_polars_params", {})
486489
for name, param_info in polars_params.items():
490+
if not param_info.is_const:
491+
continue # Skip column bindings, only extract ConstParam
487492
seen_names.add(name)
488-
# Convert Polars type to string representation
489-
type_name: str | None = None
490-
if param_info.polars_type is not None:
491-
type_name = str(param_info.polars_type)
492-
else:
493-
type_name = "any" # Dynamic type
494-
493+
# ConstParam is always required (no default support currently)
494+
type_name = "double" # TODO: Infer from annotation type
495495
parameters.append(
496496
ParameterInfo(
497497
name=name,
498498
position=param_info.position,
499499
type_name=type_name,
500500
description=param_info.doc,
501-
required=True, # Polars params are always required
501+
required=True,
502502
default=None,
503-
constraints={},
503+
constraints=None,
504504
is_table_input=False,
505-
is_varargs=param_info.varargs,
506-
is_const=param_info.is_const,
505+
is_varargs=False,
506+
is_const=True,
507507
)
508508
)
509509

vgi/worker.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -745,6 +745,18 @@ def _process_scalar_batches(
745745
if protocol_input.is_finalize:
746746
fn_log.debug("finalize_received")
747747
generator.close()
748+
# Send FINISHED status response (client expects this)
749+
empty_output = pa.RecordBatch.from_pydict(
750+
{
751+
instance.output_schema.field(0).name: pa.array(
752+
[], type=instance.output_schema.field(0).type
753+
)
754+
},
755+
schema=instance.output_schema,
756+
)
757+
finished_md = pa.KeyValueMetadata({b"vgi.status": b"FINISHED"})
758+
writer.write_batch(empty_output, custom_metadata=finished_md)
759+
fn_log.debug("finished_status_sent")
748760
# Read end-of-stream marker before returning to invocation loop
749761
try:
750762
data_reader.read_next_batch_with_custom_metadata()

0 commit comments

Comments
 (0)