Skip to content

Commit 1cfe5f0

Browse files
ferhatelmasferhatelmas
committed
fix: empty segment listing
Signed-off-by: ferhat elmas <elmas.ferhat@gmail.com>
1 parent f6e193a commit 1cfe5f0

5 files changed

Lines changed: 1212 additions & 17 deletions

File tree

‎migrations/tenant/0059-fix-common-prefix-empty-segments.sql‎

Lines changed: 351 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,351 @@
1+
CREATE OR REPLACE FUNCTION storage.get_common_prefix(
2+
p_key TEXT,
3+
p_prefix TEXT,
4+
p_delimiter TEXT
5+
) RETURNS TEXT
6+
IMMUTABLE
7+
LANGUAGE plpgsql
8+
AS $$
9+
DECLARE
10+
v_prefix TEXT := coalesce(p_prefix, '');
11+
v_suffix TEXT;
12+
v_scan_index INT := 1;
13+
v_next_delimiter_index INT;
14+
BEGIN
15+
IF coalesce(p_delimiter, '') = '' THEN
16+
RETURN NULL;
17+
END IF;
18+
19+
IF v_prefix <> '' AND lower(left(p_key, length(v_prefix))) <> lower(v_prefix) THEN
20+
RETURN NULL;
21+
END IF;
22+
23+
v_suffix := substring(p_key FROM length(v_prefix) + 1);
24+
25+
-- Only skip leading delimiters when the prefix already ends with the
26+
-- delimiter (empty-segment case like 'prefix//file'). Otherwise the
27+
-- first delimiter in the suffix is the real folder boundary.
28+
IF right(v_prefix, length(p_delimiter)) = p_delimiter THEN
29+
WHILE left(substring(v_suffix FROM v_scan_index), length(p_delimiter)) = p_delimiter LOOP
30+
v_scan_index := v_scan_index + length(p_delimiter);
31+
END LOOP;
32+
END IF;
33+
34+
v_next_delimiter_index := position(p_delimiter IN substring(v_suffix FROM v_scan_index));
35+
IF v_next_delimiter_index = 0 THEN
36+
RETURN NULL;
37+
END IF;
38+
39+
RETURN left(
40+
p_key,
41+
length(v_prefix) + (v_scan_index - 1) + v_next_delimiter_index - 1 + length(p_delimiter)
42+
);
43+
END;
44+
$$;
45+
46+
CREATE OR REPLACE FUNCTION storage.get_prefix_child_name(
47+
p_key TEXT,
48+
p_prefix TEXT,
49+
p_delimiter TEXT
50+
) RETURNS TEXT
51+
IMMUTABLE
52+
LANGUAGE plpgsql
53+
AS $$
54+
DECLARE
55+
v_prefix TEXT := coalesce(p_prefix, '');
56+
v_suffix TEXT;
57+
v_scan_index INT := 1;
58+
v_trimmed_suffix TEXT;
59+
BEGIN
60+
IF coalesce(p_delimiter, '') = '' THEN
61+
RETURN NULL;
62+
END IF;
63+
64+
IF v_prefix <> '' AND lower(left(p_key, length(v_prefix))) <> lower(v_prefix) THEN
65+
RETURN NULL;
66+
END IF;
67+
68+
v_suffix := substring(p_key FROM length(v_prefix) + 1);
69+
70+
IF right(v_prefix, length(p_delimiter)) = p_delimiter THEN
71+
WHILE left(substring(v_suffix FROM v_scan_index), length(p_delimiter)) = p_delimiter LOOP
72+
v_scan_index := v_scan_index + length(p_delimiter);
73+
END LOOP;
74+
END IF;
75+
76+
v_trimmed_suffix := substring(v_suffix FROM v_scan_index);
77+
IF coalesce(v_trimmed_suffix, '') = '' THEN
78+
RETURN NULL;
79+
END IF;
80+
81+
RETURN split_part(v_trimmed_suffix, p_delimiter, 1);
82+
END;
83+
$$;
84+
85+
CREATE OR REPLACE FUNCTION storage.search(
86+
prefix text,
87+
bucketname text,
88+
limits int DEFAULT 100,
89+
levels int DEFAULT 1,
90+
offsets int DEFAULT 0,
91+
search text DEFAULT '',
92+
sortcolumn text DEFAULT 'name',
93+
sortorder text DEFAULT 'asc'
94+
)
95+
RETURNS TABLE (
96+
name text,
97+
id uuid,
98+
updated_at timestamptz,
99+
created_at timestamptz,
100+
last_accessed_at timestamptz,
101+
metadata jsonb
102+
)
103+
SECURITY INVOKER
104+
LANGUAGE plpgsql STABLE
105+
AS $func$
106+
DECLARE
107+
v_peek_name TEXT;
108+
v_current RECORD;
109+
v_common_prefix TEXT;
110+
v_child_name TEXT;
111+
v_delimiter CONSTANT TEXT := '/';
112+
113+
-- Configuration
114+
v_limit INT;
115+
v_prefix TEXT;
116+
v_raw_prefix TEXT;
117+
v_prefix_lower TEXT;
118+
v_is_asc BOOLEAN;
119+
v_order_by TEXT;
120+
v_sort_order TEXT;
121+
v_upper_bound TEXT;
122+
v_file_batch_size INT;
123+
124+
-- Dynamic SQL for batch query only
125+
v_batch_query TEXT;
126+
127+
-- Seek state
128+
v_next_seek TEXT;
129+
v_count INT := 0;
130+
v_skipped INT := 0;
131+
v_has_pending_peek BOOLEAN := FALSE;
132+
v_emitted_folders TEXT[] := ARRAY[]::TEXT[];
133+
BEGIN
134+
v_limit := LEAST(coalesce(limits, 100), 1500);
135+
v_prefix := coalesce(prefix, '') || coalesce(search, '');
136+
-- The caller may have LIKE-escaped the prefix (e.g. \_ \%).
137+
-- Keep the escaped version for ILIKE filters, but strip the
138+
-- backslash escapes for exact-match helper functions.
139+
v_raw_prefix := replace(replace(v_prefix, '\%', '%'), '\_', '_');
140+
v_prefix_lower := lower(v_prefix);
141+
v_is_asc := lower(coalesce(sortorder, 'asc')) = 'asc';
142+
v_file_batch_size := LEAST(GREATEST(v_limit * 2, 100), 1000);
143+
144+
CASE lower(coalesce(sortcolumn, 'name'))
145+
WHEN 'name' THEN v_order_by := 'name';
146+
WHEN 'updated_at' THEN v_order_by := 'updated_at';
147+
WHEN 'created_at' THEN v_order_by := 'created_at';
148+
WHEN 'last_accessed_at' THEN v_order_by := 'last_accessed_at';
149+
ELSE v_order_by := 'name';
150+
END CASE;
151+
152+
v_sort_order := CASE WHEN v_is_asc THEN 'asc' ELSE 'desc' END;
153+
154+
IF v_order_by != 'name' THEN
155+
RETURN QUERY EXECUTE format(
156+
$sql$
157+
WITH folders AS (
158+
SELECT storage.get_prefix_child_name(objects.name, $5, '/') AS folder
159+
FROM storage.objects
160+
WHERE objects.name ILIKE $1 || '%%'
161+
AND bucket_id = $2
162+
AND storage.get_common_prefix(objects.name, $5, '/') IS NOT NULL
163+
GROUP BY folder
164+
), entries AS (
165+
SELECT folder AS "name",
166+
NULL::uuid AS id,
167+
NULL::timestamptz AS updated_at,
168+
NULL::timestamptz AS created_at,
169+
NULL::timestamptz AS last_accessed_at,
170+
NULL::jsonb AS metadata,
171+
0 AS sort_group
172+
FROM folders
173+
WHERE folder IS NOT NULL
174+
UNION ALL
175+
SELECT storage.get_prefix_child_name(objects.name, $5, '/') AS "name",
176+
id, updated_at, created_at, last_accessed_at, metadata,
177+
1 AS sort_group
178+
FROM storage.objects
179+
WHERE objects.name ILIKE $1 || '%%'
180+
AND bucket_id = $2
181+
AND storage.get_common_prefix(objects.name, $5, '/') IS NULL
182+
AND storage.get_prefix_child_name(objects.name, $5, '/') IS NOT NULL
183+
)
184+
SELECT "name", id, updated_at, created_at, last_accessed_at, metadata
185+
FROM entries
186+
ORDER BY sort_group ASC,
187+
CASE WHEN sort_group = 0 THEN "name" END %s,
188+
CASE WHEN sort_group = 1 THEN %I END %s,
189+
CASE WHEN sort_group = 1 THEN "name" END %s
190+
LIMIT $3 OFFSET $4
191+
$sql$, v_sort_order, v_order_by, v_sort_order, v_sort_order
192+
) USING v_prefix, bucketname, v_limit, offsets, v_raw_prefix;
193+
RETURN;
194+
END IF;
195+
196+
IF v_prefix_lower = '' THEN
197+
v_upper_bound := NULL;
198+
ELSIF right(v_prefix_lower, 1) = v_delimiter THEN
199+
v_upper_bound := left(v_prefix_lower, -1) || chr(ascii(v_delimiter) + 1);
200+
ELSE
201+
v_upper_bound := left(v_prefix_lower, -1) || chr(ascii(right(v_prefix_lower, 1)) + 1);
202+
END IF;
203+
204+
IF v_is_asc THEN
205+
IF v_upper_bound IS NOT NULL THEN
206+
v_batch_query := 'SELECT o.name, o.id, o.updated_at, o.created_at, o.last_accessed_at, o.metadata ' ||
207+
'FROM storage.objects o WHERE o.bucket_id = $1 AND lower(o.name) COLLATE "C" >= $2 ' ||
208+
'AND lower(o.name) COLLATE "C" < $3 ORDER BY lower(o.name) COLLATE "C" ASC LIMIT $4';
209+
ELSE
210+
v_batch_query := 'SELECT o.name, o.id, o.updated_at, o.created_at, o.last_accessed_at, o.metadata ' ||
211+
'FROM storage.objects o WHERE o.bucket_id = $1 AND lower(o.name) COLLATE "C" >= $2 ' ||
212+
'ORDER BY lower(o.name) COLLATE "C" ASC LIMIT $4';
213+
END IF;
214+
ELSE
215+
IF v_upper_bound IS NOT NULL THEN
216+
v_batch_query := 'SELECT o.name, o.id, o.updated_at, o.created_at, o.last_accessed_at, o.metadata ' ||
217+
'FROM storage.objects o WHERE o.bucket_id = $1 AND lower(o.name) COLLATE "C" < $2 ' ||
218+
'AND lower(o.name) COLLATE "C" >= $3 ORDER BY lower(o.name) COLLATE "C" DESC LIMIT $4';
219+
ELSE
220+
v_batch_query := 'SELECT o.name, o.id, o.updated_at, o.created_at, o.last_accessed_at, o.metadata ' ||
221+
'FROM storage.objects o WHERE o.bucket_id = $1 AND lower(o.name) COLLATE "C" < $2 ' ||
222+
'ORDER BY lower(o.name) COLLATE "C" DESC LIMIT $4';
223+
END IF;
224+
END IF;
225+
226+
IF v_is_asc THEN
227+
v_next_seek := v_prefix_lower;
228+
ELSE
229+
IF v_upper_bound IS NOT NULL THEN
230+
SELECT o.name INTO v_peek_name FROM storage.objects o
231+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" >= v_prefix_lower AND lower(o.name) COLLATE "C" < v_upper_bound
232+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
233+
ELSIF v_prefix_lower <> '' THEN
234+
SELECT o.name INTO v_peek_name FROM storage.objects o
235+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" >= v_prefix_lower
236+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
237+
ELSE
238+
SELECT o.name INTO v_peek_name FROM storage.objects o
239+
WHERE o.bucket_id = bucketname
240+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
241+
END IF;
242+
243+
IF v_peek_name IS NOT NULL THEN
244+
v_next_seek := lower(v_peek_name) || v_delimiter;
245+
ELSE
246+
RETURN;
247+
END IF;
248+
END IF;
249+
250+
LOOP
251+
EXIT WHEN v_count >= v_limit;
252+
253+
IF NOT v_has_pending_peek THEN
254+
IF v_is_asc THEN
255+
IF v_upper_bound IS NOT NULL THEN
256+
SELECT o.name INTO v_peek_name FROM storage.objects o
257+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" >= v_next_seek AND lower(o.name) COLLATE "C" < v_upper_bound
258+
ORDER BY lower(o.name) COLLATE "C" ASC LIMIT 1;
259+
ELSE
260+
SELECT o.name INTO v_peek_name FROM storage.objects o
261+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" >= v_next_seek
262+
ORDER BY lower(o.name) COLLATE "C" ASC LIMIT 1;
263+
END IF;
264+
ELSE
265+
IF v_upper_bound IS NOT NULL THEN
266+
SELECT o.name INTO v_peek_name FROM storage.objects o
267+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" < v_next_seek AND lower(o.name) COLLATE "C" >= v_prefix_lower
268+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
269+
ELSIF v_prefix_lower <> '' THEN
270+
SELECT o.name INTO v_peek_name FROM storage.objects o
271+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" < v_next_seek AND lower(o.name) COLLATE "C" >= v_prefix_lower
272+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
273+
ELSE
274+
SELECT o.name INTO v_peek_name FROM storage.objects o
275+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" < v_next_seek
276+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
277+
END IF;
278+
END IF;
279+
END IF;
280+
281+
v_has_pending_peek := FALSE;
282+
283+
EXIT WHEN v_peek_name IS NULL;
284+
285+
v_common_prefix := storage.get_common_prefix(lower(v_peek_name), v_prefix_lower, v_delimiter);
286+
287+
IF v_common_prefix IS NOT NULL THEN
288+
v_child_name := storage.get_prefix_child_name(v_peek_name, v_prefix, v_delimiter);
289+
290+
IF v_child_name IS NOT NULL AND array_position(v_emitted_folders, v_child_name) IS NULL THEN
291+
v_emitted_folders := array_append(v_emitted_folders, v_child_name);
292+
293+
IF v_skipped < offsets THEN
294+
v_skipped := v_skipped + 1;
295+
ELSE
296+
name := v_child_name;
297+
id := NULL;
298+
updated_at := NULL;
299+
created_at := NULL;
300+
last_accessed_at := NULL;
301+
metadata := NULL;
302+
RETURN NEXT;
303+
v_count := v_count + 1;
304+
END IF;
305+
END IF;
306+
307+
IF v_is_asc THEN
308+
v_next_seek := lower(left(v_common_prefix, -1)) || chr(ascii(v_delimiter) + 1);
309+
ELSE
310+
v_next_seek := lower(v_common_prefix);
311+
END IF;
312+
ELSE
313+
FOR v_current IN EXECUTE v_batch_query
314+
USING bucketname, v_next_seek,
315+
CASE WHEN v_is_asc THEN COALESCE(v_upper_bound, v_prefix_lower) ELSE v_prefix_lower END, v_file_batch_size
316+
LOOP
317+
v_common_prefix := storage.get_common_prefix(lower(v_current.name), v_prefix_lower, v_delimiter);
318+
319+
IF v_common_prefix IS NOT NULL THEN
320+
v_peek_name := v_current.name;
321+
v_has_pending_peek := TRUE;
322+
EXIT;
323+
END IF;
324+
325+
IF v_skipped < offsets THEN
326+
v_skipped := v_skipped + 1;
327+
ELSE
328+
name := storage.get_prefix_child_name(v_current.name, v_prefix, v_delimiter);
329+
IF name IS NOT NULL THEN
330+
id := v_current.id;
331+
updated_at := v_current.updated_at;
332+
created_at := v_current.created_at;
333+
last_accessed_at := v_current.last_accessed_at;
334+
metadata := v_current.metadata;
335+
RETURN NEXT;
336+
v_count := v_count + 1;
337+
END IF;
338+
END IF;
339+
340+
IF v_is_asc THEN
341+
v_next_seek := lower(v_current.name) || v_delimiter;
342+
ELSE
343+
v_next_seek := lower(v_current.name);
344+
END IF;
345+
346+
EXIT WHEN v_count >= v_limit;
347+
END LOOP;
348+
END IF;
349+
END LOOP;
350+
END;
351+
$func$;

‎src/internal/database/migrations/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,5 @@ export const DBMigration = {
5858
'fix-optimized-search-function': 56,
5959
's3-multipart-uploads-metadata': 57,
6060
'operation-ergonomics': 58,
61+
'fix-common-prefix-empty-segments': 59,
6162
} as const

0 commit comments

Comments
 (0)