Skip to content

Commit 0e99200

Browse files
ferhatelmasferhatelmas
committed
fix: empty segment listing
Signed-off-by: ferhat elmas <elmas.ferhat@gmail.com>
1 parent f6e193a commit 0e99200

5 files changed

Lines changed: 855 additions & 17 deletions

File tree

‎migrations/tenant/0059-fix-common-prefix-empty-segments.sql‎

Lines changed: 345 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,345 @@
1+
CREATE OR REPLACE FUNCTION storage.get_common_prefix(
2+
p_key TEXT,
3+
p_prefix TEXT,
4+
p_delimiter TEXT
5+
) RETURNS TEXT
6+
IMMUTABLE
7+
LANGUAGE plpgsql
8+
AS $$
9+
DECLARE
10+
v_prefix TEXT := coalesce(p_prefix, '');
11+
v_suffix TEXT;
12+
v_scan_index INT := 1;
13+
v_next_delimiter_index INT;
14+
BEGIN
15+
IF coalesce(p_delimiter, '') = '' THEN
16+
RETURN NULL;
17+
END IF;
18+
19+
IF v_prefix <> '' AND lower(left(p_key, length(v_prefix))) <> lower(v_prefix) THEN
20+
RETURN NULL;
21+
END IF;
22+
23+
v_suffix := substring(p_key FROM length(v_prefix) + 1);
24+
25+
-- Only skip leading delimiters when the prefix already ends with the
26+
-- delimiter (empty-segment case like 'prefix//file'). Otherwise the
27+
-- first delimiter in the suffix is the real folder boundary.
28+
IF right(v_prefix, length(p_delimiter)) = p_delimiter THEN
29+
WHILE left(substring(v_suffix FROM v_scan_index), length(p_delimiter)) = p_delimiter LOOP
30+
v_scan_index := v_scan_index + length(p_delimiter);
31+
END LOOP;
32+
END IF;
33+
34+
v_next_delimiter_index := position(p_delimiter IN substring(v_suffix FROM v_scan_index));
35+
IF v_next_delimiter_index = 0 THEN
36+
RETURN NULL;
37+
END IF;
38+
39+
RETURN left(
40+
p_key,
41+
length(v_prefix) + (v_scan_index - 1) + v_next_delimiter_index - 1 + length(p_delimiter)
42+
);
43+
END;
44+
$$;
45+
46+
CREATE OR REPLACE FUNCTION storage.get_prefix_child_name(
47+
p_key TEXT,
48+
p_prefix TEXT,
49+
p_delimiter TEXT
50+
) RETURNS TEXT
51+
IMMUTABLE
52+
LANGUAGE plpgsql
53+
AS $$
54+
DECLARE
55+
v_prefix TEXT := coalesce(p_prefix, '');
56+
v_suffix TEXT;
57+
v_scan_index INT := 1;
58+
v_trimmed_suffix TEXT;
59+
BEGIN
60+
IF coalesce(p_delimiter, '') = '' THEN
61+
RETURN NULL;
62+
END IF;
63+
64+
IF v_prefix <> '' AND lower(left(p_key, length(v_prefix))) <> lower(v_prefix) THEN
65+
RETURN NULL;
66+
END IF;
67+
68+
v_suffix := substring(p_key FROM length(v_prefix) + 1);
69+
70+
IF right(v_prefix, length(p_delimiter)) = p_delimiter THEN
71+
WHILE left(substring(v_suffix FROM v_scan_index), length(p_delimiter)) = p_delimiter LOOP
72+
v_scan_index := v_scan_index + length(p_delimiter);
73+
END LOOP;
74+
END IF;
75+
76+
v_trimmed_suffix := substring(v_suffix FROM v_scan_index);
77+
IF coalesce(v_trimmed_suffix, '') = '' THEN
78+
RETURN NULL;
79+
END IF;
80+
81+
RETURN split_part(v_trimmed_suffix, p_delimiter, 1);
82+
END;
83+
$$;
84+
85+
CREATE OR REPLACE FUNCTION storage.search(
86+
prefix text,
87+
bucketname text,
88+
limits int DEFAULT 100,
89+
levels int DEFAULT 1,
90+
offsets int DEFAULT 0,
91+
search text DEFAULT '',
92+
sortcolumn text DEFAULT 'name',
93+
sortorder text DEFAULT 'asc'
94+
)
95+
RETURNS TABLE (
96+
name text,
97+
id uuid,
98+
updated_at timestamptz,
99+
created_at timestamptz,
100+
last_accessed_at timestamptz,
101+
metadata jsonb
102+
)
103+
SECURITY INVOKER
104+
LANGUAGE plpgsql STABLE
105+
AS $func$
106+
DECLARE
107+
v_peek_name TEXT;
108+
v_current RECORD;
109+
v_common_prefix TEXT;
110+
v_delimiter CONSTANT TEXT := '/';
111+
112+
-- Configuration
113+
v_limit INT;
114+
v_prefix TEXT;
115+
v_raw_prefix TEXT;
116+
v_prefix_lower TEXT;
117+
v_is_asc BOOLEAN;
118+
v_order_by TEXT;
119+
v_sort_order TEXT;
120+
v_upper_bound TEXT;
121+
v_file_batch_size INT;
122+
123+
-- Dynamic SQL for batch query only
124+
v_batch_query TEXT;
125+
126+
-- Seek state
127+
v_next_seek TEXT;
128+
v_count INT := 0;
129+
v_skipped INT := 0;
130+
v_has_pending_peek BOOLEAN := FALSE;
131+
BEGIN
132+
v_limit := LEAST(coalesce(limits, 100), 1500);
133+
v_prefix := coalesce(prefix, '') || coalesce(search, '');
134+
-- The caller may have LIKE-escaped the prefix (e.g. \_ \%).
135+
-- Keep the escaped version for ILIKE filters, but strip the
136+
-- backslash escapes for exact-match helper functions.
137+
v_raw_prefix := replace(replace(v_prefix, '\%', '%'), '\_', '_');
138+
v_prefix_lower := lower(v_prefix);
139+
v_is_asc := lower(coalesce(sortorder, 'asc')) = 'asc';
140+
v_file_batch_size := LEAST(GREATEST(v_limit * 2, 100), 1000);
141+
142+
CASE lower(coalesce(sortcolumn, 'name'))
143+
WHEN 'name' THEN v_order_by := 'name';
144+
WHEN 'updated_at' THEN v_order_by := 'updated_at';
145+
WHEN 'created_at' THEN v_order_by := 'created_at';
146+
WHEN 'last_accessed_at' THEN v_order_by := 'last_accessed_at';
147+
ELSE v_order_by := 'name';
148+
END CASE;
149+
150+
v_sort_order := CASE WHEN v_is_asc THEN 'asc' ELSE 'desc' END;
151+
152+
IF v_order_by != 'name' THEN
153+
RETURN QUERY EXECUTE format(
154+
$sql$
155+
WITH folders AS (
156+
SELECT storage.get_prefix_child_name(objects.name, $5, '/') AS folder
157+
FROM storage.objects
158+
WHERE objects.name ILIKE $1 || '%%'
159+
AND bucket_id = $2
160+
AND storage.get_common_prefix(objects.name, $5, '/') IS NOT NULL
161+
GROUP BY folder
162+
), entries AS (
163+
SELECT folder AS "name",
164+
NULL::uuid AS id,
165+
NULL::timestamptz AS updated_at,
166+
NULL::timestamptz AS created_at,
167+
NULL::timestamptz AS last_accessed_at,
168+
NULL::jsonb AS metadata,
169+
0 AS sort_group
170+
FROM folders
171+
WHERE folder IS NOT NULL
172+
UNION ALL
173+
SELECT storage.get_prefix_child_name(objects.name, $5, '/') AS "name",
174+
id, updated_at, created_at, last_accessed_at, metadata,
175+
1 AS sort_group
176+
FROM storage.objects
177+
WHERE objects.name ILIKE $1 || '%%'
178+
AND bucket_id = $2
179+
AND storage.get_common_prefix(objects.name, $5, '/') IS NULL
180+
AND storage.get_prefix_child_name(objects.name, $5, '/') IS NOT NULL
181+
)
182+
SELECT "name", id, updated_at, created_at, last_accessed_at, metadata
183+
FROM entries
184+
ORDER BY sort_group ASC,
185+
CASE WHEN sort_group = 0 THEN "name" END %s,
186+
CASE WHEN sort_group = 1 THEN %I END %s,
187+
CASE WHEN sort_group = 1 THEN "name" END %s
188+
LIMIT $3 OFFSET $4
189+
$sql$, v_sort_order, v_order_by, v_sort_order, v_sort_order
190+
) USING v_prefix, bucketname, v_limit, offsets, v_raw_prefix;
191+
RETURN;
192+
END IF;
193+
194+
IF v_prefix_lower = '' THEN
195+
v_upper_bound := NULL;
196+
ELSIF right(v_prefix_lower, 1) = v_delimiter THEN
197+
v_upper_bound := left(v_prefix_lower, -1) || chr(ascii(v_delimiter) + 1);
198+
ELSE
199+
v_upper_bound := left(v_prefix_lower, -1) || chr(ascii(right(v_prefix_lower, 1)) + 1);
200+
END IF;
201+
202+
IF v_is_asc THEN
203+
IF v_upper_bound IS NOT NULL THEN
204+
v_batch_query := 'SELECT o.name, o.id, o.updated_at, o.created_at, o.last_accessed_at, o.metadata ' ||
205+
'FROM storage.objects o WHERE o.bucket_id = $1 AND lower(o.name) COLLATE "C" >= $2 ' ||
206+
'AND lower(o.name) COLLATE "C" < $3 ORDER BY lower(o.name) COLLATE "C" ASC LIMIT $4';
207+
ELSE
208+
v_batch_query := 'SELECT o.name, o.id, o.updated_at, o.created_at, o.last_accessed_at, o.metadata ' ||
209+
'FROM storage.objects o WHERE o.bucket_id = $1 AND lower(o.name) COLLATE "C" >= $2 ' ||
210+
'ORDER BY lower(o.name) COLLATE "C" ASC LIMIT $4';
211+
END IF;
212+
ELSE
213+
IF v_upper_bound IS NOT NULL THEN
214+
v_batch_query := 'SELECT o.name, o.id, o.updated_at, o.created_at, o.last_accessed_at, o.metadata ' ||
215+
'FROM storage.objects o WHERE o.bucket_id = $1 AND lower(o.name) COLLATE "C" < $2 ' ||
216+
'AND lower(o.name) COLLATE "C" >= $3 ORDER BY lower(o.name) COLLATE "C" DESC LIMIT $4';
217+
ELSE
218+
v_batch_query := 'SELECT o.name, o.id, o.updated_at, o.created_at, o.last_accessed_at, o.metadata ' ||
219+
'FROM storage.objects o WHERE o.bucket_id = $1 AND lower(o.name) COLLATE "C" < $2 ' ||
220+
'ORDER BY lower(o.name) COLLATE "C" DESC LIMIT $4';
221+
END IF;
222+
END IF;
223+
224+
IF v_is_asc THEN
225+
v_next_seek := v_prefix_lower;
226+
ELSE
227+
IF v_upper_bound IS NOT NULL THEN
228+
SELECT o.name INTO v_peek_name FROM storage.objects o
229+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" >= v_prefix_lower AND lower(o.name) COLLATE "C" < v_upper_bound
230+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
231+
ELSIF v_prefix_lower <> '' THEN
232+
SELECT o.name INTO v_peek_name FROM storage.objects o
233+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" >= v_prefix_lower
234+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
235+
ELSE
236+
SELECT o.name INTO v_peek_name FROM storage.objects o
237+
WHERE o.bucket_id = bucketname
238+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
239+
END IF;
240+
241+
IF v_peek_name IS NOT NULL THEN
242+
v_next_seek := lower(v_peek_name) || v_delimiter;
243+
ELSE
244+
RETURN;
245+
END IF;
246+
END IF;
247+
248+
LOOP
249+
EXIT WHEN v_count >= v_limit;
250+
251+
IF NOT v_has_pending_peek THEN
252+
IF v_is_asc THEN
253+
IF v_upper_bound IS NOT NULL THEN
254+
SELECT o.name INTO v_peek_name FROM storage.objects o
255+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" >= v_next_seek AND lower(o.name) COLLATE "C" < v_upper_bound
256+
ORDER BY lower(o.name) COLLATE "C" ASC LIMIT 1;
257+
ELSE
258+
SELECT o.name INTO v_peek_name FROM storage.objects o
259+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" >= v_next_seek
260+
ORDER BY lower(o.name) COLLATE "C" ASC LIMIT 1;
261+
END IF;
262+
ELSE
263+
IF v_upper_bound IS NOT NULL THEN
264+
SELECT o.name INTO v_peek_name FROM storage.objects o
265+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" < v_next_seek AND lower(o.name) COLLATE "C" >= v_prefix_lower
266+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
267+
ELSIF v_prefix_lower <> '' THEN
268+
SELECT o.name INTO v_peek_name FROM storage.objects o
269+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" < v_next_seek AND lower(o.name) COLLATE "C" >= v_prefix_lower
270+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
271+
ELSE
272+
SELECT o.name INTO v_peek_name FROM storage.objects o
273+
WHERE o.bucket_id = bucketname AND lower(o.name) COLLATE "C" < v_next_seek
274+
ORDER BY lower(o.name) COLLATE "C" DESC LIMIT 1;
275+
END IF;
276+
END IF;
277+
END IF;
278+
279+
v_has_pending_peek := FALSE;
280+
281+
EXIT WHEN v_peek_name IS NULL;
282+
283+
v_common_prefix := storage.get_common_prefix(lower(v_peek_name), v_prefix_lower, v_delimiter);
284+
285+
IF v_common_prefix IS NOT NULL THEN
286+
IF v_skipped < offsets THEN
287+
v_skipped := v_skipped + 1;
288+
ELSE
289+
name := storage.get_prefix_child_name(v_peek_name, v_prefix, v_delimiter);
290+
IF name IS NOT NULL THEN
291+
id := NULL;
292+
updated_at := NULL;
293+
created_at := NULL;
294+
last_accessed_at := NULL;
295+
metadata := NULL;
296+
RETURN NEXT;
297+
v_count := v_count + 1;
298+
END IF;
299+
END IF;
300+
301+
IF v_is_asc THEN
302+
v_next_seek := lower(left(v_common_prefix, -1)) || chr(ascii(v_delimiter) + 1);
303+
ELSE
304+
v_next_seek := lower(v_common_prefix);
305+
END IF;
306+
ELSE
307+
FOR v_current IN EXECUTE v_batch_query
308+
USING bucketname, v_next_seek,
309+
CASE WHEN v_is_asc THEN COALESCE(v_upper_bound, v_prefix_lower) ELSE v_prefix_lower END, v_file_batch_size
310+
LOOP
311+
v_common_prefix := storage.get_common_prefix(lower(v_current.name), v_prefix_lower, v_delimiter);
312+
313+
IF v_common_prefix IS NOT NULL THEN
314+
v_peek_name := v_current.name;
315+
v_has_pending_peek := TRUE;
316+
EXIT;
317+
END IF;
318+
319+
IF v_skipped < offsets THEN
320+
v_skipped := v_skipped + 1;
321+
ELSE
322+
name := storage.get_prefix_child_name(v_current.name, v_prefix, v_delimiter);
323+
IF name IS NOT NULL THEN
324+
id := v_current.id;
325+
updated_at := v_current.updated_at;
326+
created_at := v_current.created_at;
327+
last_accessed_at := v_current.last_accessed_at;
328+
metadata := v_current.metadata;
329+
RETURN NEXT;
330+
v_count := v_count + 1;
331+
END IF;
332+
END IF;
333+
334+
IF v_is_asc THEN
335+
v_next_seek := lower(v_current.name) || v_delimiter;
336+
ELSE
337+
v_next_seek := lower(v_current.name);
338+
END IF;
339+
340+
EXIT WHEN v_count >= v_limit;
341+
END LOOP;
342+
END IF;
343+
END LOOP;
344+
END;
345+
$func$;

‎src/internal/database/migrations/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,5 @@ export const DBMigration = {
5858
'fix-optimized-search-function': 56,
5959
's3-multipart-uploads-metadata': 57,
6060
'operation-ergonomics': 58,
61+
'fix-common-prefix-empty-segments': 59,
6162
} as const

0 commit comments

Comments
 (0)