-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathstreamlit_app.py
More file actions
283 lines (250 loc) · 11.7 KB
/
streamlit_app.py
File metadata and controls
283 lines (250 loc) · 11.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
import requests
import time
import zipfile
import io
import streamlit as st
import json
def get_access_token(username, password, client_id, client_secret):
url = "https://vantage-au.abbyy.com/auth2/connect/token"
data = {
"grant_type": "password",
"scope": "openid permissions global.wildcard",
"username": username,
"password": password,
"client_id": client_id,
"client_secret": client_secret
}
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
response = requests.post(url, headers=headers, data=data)
if response.status_code == 200:
return response.json()["access_token"]
else:
print("Error:", response.status_code, response.text)
return None
def create_transaction(access_token):
api_url = 'https://vantage-au.abbyy.com/api/publicapi/v1/transactions'
data = {
"skillId": "e9e6517a-7a19-4f73-b2e4-857227f904f4",
"generateMobileInputLink": False,
"registrationParameters": []
}
headers = {'accept': 'text/plain', 'Authorization': f'Bearer {access_token}'}
response = requests.post(api_url, headers=headers, json=data)
return response.json()['transactionId']
def upload_file(transaction_id, access_token, file):
api_url = f'https://vantage-au.abbyy.com/api/publicapi/v1/transactions/{transaction_id}/files'
headers = {'accept': '*/*', 'Authorization': f'Bearer {access_token}'}
model = {'Model': (None, '{"files": [{"index": 0,"imageProcessingOptions": {"autoCrop": "Default","autoOrientation": "Default"},"registrationParameters": []}]}' )}
file_data = {
'file': (file.name, file.read(), 'application/pdf')
}
response = requests.post(api_url, headers=headers, data=model, files=file_data)
return response
def start_transactions(transaction_id, access_token):
api_url = f'https://vantage-au.abbyy.com/api/publicapi/v1/transactions/{transaction_id}/start'
headers = {'Authorization': f'Bearer {access_token}'}
response = requests.post(api_url, headers=headers)
return response
def wait_for_processing(transaction_id, access_token):
api_url = f'https://vantage-au.abbyy.com/api/publicapi/v1/transactions/{transaction_id}'
headers = {'Authorization': f'Bearer {access_token}'}
# Add timeout mechanism to avoid infinite waiting
start_time = time.time()
timeout = 300 # 5 minutes timeout
while True:
try:
response = requests.get(api_url, headers=headers, timeout=30)
if response.status_code == 200:
status = response.json()['status']
if status == 'Processed':
break
# Check if timeout occurred
if time.time() - start_time > timeout:
st.error("Processing timed out, please try again")
return False
time.sleep(5)
except requests.RequestException as e:
print(f"Request exception: {e}")
st.error(f"Request exception: {e}")
# Give up after timeout
if time.time() - start_time > timeout:
return False
time.sleep(5)
return True
def download_processed_files(transaction_id, access_token):
api_url = f'https://vantage-au.abbyy.com/api/publicapi/v1/transactions/{transaction_id}'
headers = {'Authorization': f'Bearer {access_token}'}
try:
response = requests.get(api_url, headers=headers, timeout=30)
if response.status_code == 200:
return response.json()['documents']
else:
st.error(f"Failed to retrieve documents: {response.status_code} - {response.text}")
return []
except Exception as e:
st.error(f"Error downloading processed files: {str(e)}")
return []
def get_bill_of_lading(result):
try:
return result['Fields']['Bill of Lading']
except (KeyError, TypeError) as e:
print(f"Warning: 'Bill of Lading' field not found: {str(e)}")
return "not_bill_of_lading"
def download_file(file_id, transaction_id, access_token):
api_url = f'https://vantage-au.abbyy.com/api/publicapi/v1/transactions/{transaction_id}/files/{file_id}/download'
headers = {'Authorization': f'Bearer {access_token}'}
try:
response = requests.get(api_url, headers=headers, timeout=60)
if response.status_code == 200:
return response.content
else:
print("Error:", response.status_code, response.text)
st.error(f"Failed to download file: {response.status_code} - {response.text}")
return None
except Exception as e:
print(f"File download exception: {str(e)}")
st.error(f"File download exception: {str(e)}")
return None
def process_pdf_file(uploaded_file):
access_token = get_access_token(st.secrets["username_abby"], st.secrets["password_abby"],
st.secrets["CLIENT_ID"], st.secrets["CLIENT_SECRET"])
if not access_token:
st.error("Failed to get access token.")
return
transaction_id = create_transaction(access_token)
if not transaction_id:
st.error("Failed to create transaction.")
return
response = upload_file(transaction_id, access_token, uploaded_file)
if response.status_code != 200:
st.error(f"Failed to upload file: {response.status_code} - {response.text}")
return
start_transactions(transaction_id, access_token)
# Check if processing completed successfully
if not wait_for_processing(transaction_id, access_token):
st.error("File processing did not complete successfully")
return
documents = download_processed_files(transaction_id, access_token)
if not documents:
st.error("No processed documents found")
return
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file:
# Track used filenames to avoid duplicates
used_filenames = {}
for document in documents:
try:
# Get JSON content
if len(document.get('resultFiles', [])) >= 1:
file_id_json = document['resultFiles'][0]['fileId']
json_content = download_file(file_id_json, transaction_id, access_token)
if json_content:
try:
result = json.loads(json_content)
bill_of_lading = get_bill_of_lading(result)
except json.JSONDecodeError:
st.error("JSON parsing error, using default filename")
bill_of_lading = "not_bill_of_lading"
else:
bill_of_lading = "not_bill_of_lading"
# Get PDF content
if len(document.get('resultFiles', [])) >= 2:
file_id_pdf = document['resultFiles'][1]['fileId']
pdf_content = download_file(file_id_pdf, transaction_id, access_token)
if pdf_content:
# Handle filenames to avoid duplicates
base_filename = f"{bill_of_lading}.pdf"
if base_filename in used_filenames:
# If filename exists, add index
used_filenames[base_filename] += 1
filename_parts = base_filename.split('.')
indexed_filename = f"{'.'.join(filename_parts[:-1])}_{used_filenames[base_filename]}.{filename_parts[-1]}"
zip_file.writestr(indexed_filename, pdf_content)
else:
# First time using this filename
used_filenames[base_filename] = 0
zip_file.writestr(base_filename, pdf_content)
except (KeyError, IndexError) as e:
st.error(f"Error processing document: {str(e)}")
continue
zip_buffer.seek(0)
return zip_buffer
def process_pdf_files(uploaded_files):
if len(uploaded_files) == 1:
zip_filename = uploaded_files[0].name.replace(".pdf", ".zip")
zip_buffer = process_pdf_file(uploaded_files[0])
if zip_buffer:
st.download_button(
label=f"Download '{zip_filename}' file",
data=zip_buffer.getvalue(),
file_name=zip_filename,
mime="application/zip"
)
else:
zip_buffer_all = io.BytesIO()
# Track used filenames in outer ZIP
outer_used_filenames = {}
with zipfile.ZipFile(zip_buffer_all, 'w', zipfile.ZIP_DEFLATED) as zip_file_all:
for i, uploaded_file in enumerate(uploaded_files):
zip_buffer = process_pdf_file(uploaded_file)
if zip_buffer:
base_filename = uploaded_file.name.replace(".pdf", ".zip")
# Handle duplicate filenames in outer ZIP
if base_filename in outer_used_filenames:
outer_used_filenames[base_filename] += 1
filename_parts = base_filename.split('.')
indexed_filename = f"{'.'.join(filename_parts[:-1])}_{outer_used_filenames[base_filename]}.{filename_parts[-1]}"
zip_file_all.writestr(indexed_filename, zip_buffer.getvalue())
else:
outer_used_filenames[base_filename] = 0
zip_file_all.writestr(base_filename, zip_buffer.getvalue())
if zip_buffer_all.getbuffer().nbytes > 0: # Check if ZIP is empty
zip_filename = "all_zips.zip"
st.download_button(
label="Download All ZIPs",
data=zip_buffer_all.getvalue(),
file_name=zip_filename,
mime="application/zip"
)
else:
st.error("Failed to process files, no downloadable content generated")
import hmac
import streamlit as st
def check_password():
"""Returns `True` if the user had a correct password."""
def login_form():
"""Form with widgets to collect user information"""
with st.form("Credentials"):
st.text_input("Username", key="username")
st.text_input("Password", type="password", key="password")
st.form_submit_button("Log in", on_click=password_entered)
def password_entered():
"""Checks whether a password entered by the user is correct."""
if st.session_state["username"] in st.secrets[
"passwords"
] and hmac.compare_digest(
st.session_state["password"],
st.secrets.passwords[st.session_state["username"]],
):
st.session_state["password_correct"] = True
del st.session_state["password"] # Don't store the username or password.
del st.session_state["username"]
else:
st.session_state["password_correct"] = False
# Return True if the username + password is validated.
if st.session_state.get("password_correct", False):
return True
# Show inputs for username + password.
login_form()
if "password_correct" in st.session_state:
st.error("😕 User not known or password incorrect")
return False
if not check_password():
st.stop()
st.set_page_config(page_title="Split pdf files by bill of lading No", layout="wide")
st.sidebar.header("Split pdf files by bill of lading No")
with st.sidebar.form("pdfselectForm", clear_on_submit=True):
uploaded_files = st.file_uploader("Choose PDF files", accept_multiple_files=True, type="pdf")
submit_button = st.form_submit_button("Split pdf files")
if submit_button and uploaded_files:
process_pdf_files(uploaded_files)