json_to_csv.py

import json
import os
import re
import sys
import csv
import io

ENCODING = "utf-8"

def identify_groups(keys):
    groups = get_groups()
    return list(set(groups).intersection(set(keys)))

def get_groups():
    # generated by the R package! sort(unique(fieldsdf$group))
    groups= [
        "applicants", "application", "assignee_years", "assignees", "attorneys",
        "botanic", "cpc_at_issue", "cpc_classes", "cpc_current", "cpc_groups",
         "cpc_subclasses", "examiners", "figures", "foreign_citations", "foreign_priority",
         "g_brf_sum_texts", "g_claims", "g_detail_desc_texts", "g_draw_desc_texts",
         "gov_interest_contract_award_numbers", "gov_interest_organizations",
         "granted_pregrant_crosswalk", "inventor_years", "inventors", "ipcr", "locations",
         "other_references", "patents", "pct_data", "pg_brf_sum_texts", "pg_claims",
         "pg_detail_desc_texts", "pg_draw_desc_texts", "publications", "rel_app_texts",
         "us_application_citations", "us_parties", "us_patent_citations", "us_related_documents",
         "us_term_of_grant", "uspc_at_issue", "uspc_mainclasses", "uspc_subclasses", "wipo"
    ]

    return groups

def convertToCSV(jsonData, keys):
    returnData = {}
    global counter
    row = []

    common = identify_groups(keys)

    if(len(common)>0):
        # Generate the length of maximum results
        length_dict = len(jsonData[common[0]])  
        for group in common:
            if len(jsonData[group]) > length_dict:
                length_dict = len(jsonData[group])
    else:  
        length_dict = 1
    for i in range(0, length_dict):
        row = []
        returnData[i] = {}
        for key in keys:
            if key in common:
                try:
                    index = keys.index(key)
                    tempData = jsonData[key][i]
                    tempKeys = sorted(tempData.keys())
                    for k in tempKeys:
                        returnData[i][k] = tempData[k]
                except:
                    pass
            else:
                returnData[i][key] = jsonData[key]
    return returnData
 
def writeCSV(a, filename):
    write = csv.writer(io.open(filename, 'w', newline='', encoding=ENCODING))
    key = identify_groups(list(a.keys()))

    j = a[key[0]]
    i = 0
    prevRow = []
    if (j is not None):
        for jsonData in j:
            k = 0
            keys = jsonData.keys()
            csvData = convertToCSV(jsonData, sorted(keys))
            if (i==0):
                write.writerow(list(sorted(csvData[0].keys())))
            for key in csvData.keys():
                row = []
                row2 = []
                data = csvData[key]
                for k in sorted(csvData[0].keys()):
                    try:
                        row = row + [data[k]]
                    except:
                        row = row + [csvData[0][k]]
                flag = False
                for item in row:
                    if item != "":
                        flag = True
                if (flag):
                    try:
                        row = [str(s).encode(ENCODING, "replace").decode(ENCODING, errors='ignore') for s in row]
                    except:
                        pass
                    write.writerow(row)
            i += 1
 
def merge_csv(fd,q,requests):
    diri = [d for d in os.listdir(fd) if re.search(q+'_\d+.csv',d)]
    csv_out = open(os.path.join(fd, q+'.csv'), 'w', encoding=ENCODING)
    for line in open(os.path.join(fd,q+'_0.csv'), 'rb').read().decode(ENCODING, errors='ignore'):
        csv_out.write(line)
    for i in range(requests):
        f = open(os.path.join(fd, q+'_'+str(i)+'.csv'), 'r+', encoding=ENCODING)
        next(f)
        for line in f:
            csv_out.write(line)
        f.close()
    csv_out.close()
 
def main(fd, q, requests):
    diri = [d for d in os.listdir(fd) if re.search(q+'_\d+.json',d)]
    for d in diri:
        filename = fd + '/' + d
        data = open(filename, "r", encoding=ENCODING).read()
        try:
            b = json.loads(data)
        except Exception as e:
            print("ERROR", e)
            sys.exit(1)
        filename = filename.replace('.json', '.csv')
        writeCSV(b, filename)
    merge_csv(fd, q, requests)

    # Remove individual component files
    for d in diri:
        os.remove(os.path.join(fd,d))
        os.remove(os.path.join(fd,d.replace('.json','.csv')))