profile_collector(1).ps

[christianheitkamp12-boop]

import random
import time
import json
import logging

import apiclient

import GoogleAPI.constants as constants
import GoogleAPI.plus_api as plus_api
import GoogleAPI.people_api as people_api

def get_google_plus_url(profile_id):
return 'https://plus.google.com/' + profile_id

def query_generator():
print('starting query generator...')

russian_alphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
english_alphabet = 'abcdefghijklmnopqrstuvwxyz'
digits = '0123456789'
for char in english_alphabet:
    yield char
for char in russian_alphabet:
    yield char
for char in digits:
    yield char
for first_char in english_alphabet:
    for second_char in english_alphabet:
        yield first_char + second_char
for first_char in russian_alphabet:
    for second_char in russian_alphabet:
        yield first_char + second_char

yield str(random.randint(0, 10000000))

def unique_profile_id_generator(credentials_type=constants.API_KEY_CREDENTIALS_TYPE, verbose=False):
print('searching for unique Google profile IDs...')
print('starting unique profile id generator...')
start_time = time.time()
current_time = start_time

unique_profile_ids = set()

plus_service = plus_api.get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()

queries = query_generator()
query = next(queries)
next_page_token = ''
count = 0
MAX_RESULTS = 50

while True:
    previous_time = current_time

    people_document = people_resource.search(query=query, maxResults=MAX_RESULTS,
                                             pageToken=next_page_token).execute()
    profile_ids = [item['id'] for item in people_document['items']]

    next_page_token = people_document['nextPageToken']
    if next_page_token == '':
        query = next(queries)

    previous_count = count

    for profile_id in profile_ids:
        if not (profile_id in unique_profile_ids):
            yield profile_id
            unique_profile_ids.add(profile_id)
            count += 1

    time.sleep(1)

    current_time = time.time()

    if verbose:
        print('{:<10} {} | +{} | query: "{}" | nextPageToken: "{}"'.format('ids', count, count - previous_count,
                                                                           query, next_page_token))
        # if verbose:
        #     print('{:<10} {:>10}s * {:>5}s * '
        #           '{} | +{} | query: "{}" | nextPageToken: "{}"'.format('ids', round(current_time - start_time, 2),
        #                                                                 round(current_time - previous_time, 2),
        #                                                                 count, count - previous_count, query,
        #                                                                 next_page_token))

def profile_generator(credentials_type=constants.API_KEY_CREDENTIALS_TYPE, verbose=False):
print('getting profiles through {}...'.format(credentials_type))

people_service = people_api.get_people_service(credentials_type=credentials_type)
people_resource = people_service.people()

unique_profile_ids = unique_profile_id_generator(credentials_type=credentials_type, verbose=verbose)
current_count = 0

BATCH_SIZE = 50

while True:
    resource_names = ['people/' + next(unique_profile_ids) for _ in range(BATCH_SIZE)]

    # trying to get people document
    delay = 0.5
    people_document = None
    while people_document is None:
        time.sleep(delay)
        try:
            people_document = people_resource.getBatchGet(resourceNames=resource_names,
                                                          personFields='names,photos').execute()
        except apiclient.errors.HttpError:
            delay = 2
            print('{} | Error [apiclient.errors.HttpError]. '
                  'Cannot get people document through getBatchGet.'.format(current_count))
            continue

    previous_count = current_count

    for response in people_document['responses']:
        status = response['status']
        if status != {}:
            continue

        person = response.get('person', None)
        if person is None:
            continue

        names = person.get('names', None)
        if names is None:
            continue

        photos = person.get('photos', None)
        if photos is None:
            continue

        display_name = names[0]['displayName']
        image_url = photos[0]['url']
        resource_name = response['requestedResourceName']  # 'people/114822392780451536009'
        profile_id = resource_name.split('/')[1]  # '114822392780451536009'
        google_plus_url = get_google_plus_url(profile_id)

        yield (profile_id, google_plus_url, display_name, image_url)

        current_count += 1

    if verbose:
        print('{:<10} {} | +{}'.format('profiles', current_count, current_count - previous_count))

def get_profiles(count, credentials_type=constants.API_KEY_CREDENTIALS_TYPE, verbose=False):
print('getting profiles through {}...'.format(credentials_type))
start_time = time.time()
current_time = start_time

result = []

people_service = people_api.get_people_service(credentials_type=credentials_type)
people_resource = people_service.people()

unique_profile_ids = unique_profile_id_generator(credentials_type=credentials_type, verbose=verbose)
current_count = 0

MAX_BATCH_SIZE = 50

while current_count < count:
    previous_time = current_time

    batch_size = min(MAX_BATCH_SIZE, count - current_count)
    resource_names = ['people/' + next(unique_profile_ids) for _ in range(batch_size)]

    # trying to get people document
    delay = 0.5
    people_document = None
    while people_document is None:
        time.sleep(delay)
        try:
            people_document = people_resource.getBatchGet(resourceNames=resource_names,
                                                          personFields='names,photos').execute()
        except apiclient.errors.HttpError:
            delay = 2
            print('{} | Error [apiclient.errors.HttpError]. '
                  'Cannot get people document through getBatchGet.'.format(current_count))
            continue

    previous_count = current_count

    for response in people_document['responses']:
        status = response['status']
        if status != {}:
            print('status is empty')
            continue

        person = response.get('person', None)
        if person is None:
            print('person is empty')
            continue

        names = person.get('names', None)
        if names is None:
            print('names is empty')
            continue

        photos = person.get('photos', None)
        if photos is None:
            print('photos is empty')
            continue

        display_name = names[0]['displayName']
        image_url = photos[0]['url']
        resource_name = response['requestedResourceName']  # 'people/114822392780451536009'
        profile_id = resource_name.split('/')[1]  # '114822392780451536009'
        google_plus_url = get_google_plus_url(profile_id)

        result.append((profile_id, google_plus_url, display_name, image_url))

        current_count += 1

    current_time = time.time()

    if verbose:
        print('{:<10} {:>10}s * {:>5}s * {} | +{}'.format('profiles', round(current_time - start_time, 2),
                                                          round(current_time - previous_time, 2), current_count,
                                                          current_count - previous_count))

return result

def save_profiles_as_txt(profiles, filename):
print('saving profiles as json to {}...'.format(filename))

result_str = ''
for profile in profiles:
    result_str += '\n'.join(profile) + '\n\n'
profiles_file = open(filename, 'w', encoding='utf-8')
profiles_file.write(result_str)
profiles_file.close()

def save_profiles_as_json(profiles, filename):
print('saving profiles as json to {}...'.format(filename))

result = []
JSON_FIELDS = ('profileId', 'profileUrl', 'displayName', 'imageUrl')
for profile in profiles:
    result.append(dict(zip(JSON_FIELDS, profile)))

json_str = json.dumps(result)
profiles_file = open(filename, 'w', encoding='utf-8')
profiles_file.write(json_str)
profiles_file.close()

if name == 'main':
# profiles = get_profiles(3000, credentials_type=constants.API_KEY_CREDENTIALS_TYPE, verbose=True)
profile_gen = profile_generator(credentials_type=constants.API_KEY_CREDENTIALS_TYPE, verbose=True)
profiles = [next(profile_gen) for _ in range(100)]

for profile in profiles[-10:]:
    print(profile)

save_profiles_as_txt(profiles, 'results/txts/profiles.txt')
save_profiles_as_json(profiles, 'results/jsons/profiles.json')

[christianheitkamp12-boop]

import apiclient
import oauth2client
import httplib2
import time
import random
import json

import GoogleAPI.constants as constants
import GoogleAPI.file_handler as file_handler

import GoogleAPI.plus_api

from plus_api import get_unique_profile_ids, profile_id_to_google_plus_url

def get_OAuth_2_credentials():
store = oauth2client.file.Storage(constants.OAUTH_2_CREDENTIALS_PATH)
credentials = store.get()
if not credentials or credentials.invalid:
flow = oauth2client.client.flow_from_clientsecrets(constants.CLIENT_SECRET_PATH, constants.SCOPES)
flow.user_agent = constants.APPLICATION_NAME
credentials = oauth2client.tools.run_flow(flow, store)
print('Storing credentials to ' + constants.OAUTH_2_CREDENTIALS_PATH)
return credentials

def get_people_service_through_API_key():
api_key_file = open(constants.API_KEY_CREDENTIALS_PATH, 'r', encoding='utf-8')
api_key = api_key_file.read().strip()
return apiclient.discovery.build('people', 'v1', developerKey=api_key)

def get_people_service_through_OAuth_2():
credentials = get_OAuth_2_credentials()
http = credentials.authorize(httplib2.Http())
return apiclient.discovery.build('people', 'v1', http=http)
# ,discoveryServiceUrl='https://people.googleapis.com/$discovery/rest')

def get_people_service(credentials_type=constants.API_KEY_CREDENTIALS_TYPE):
print('getting People API service through {}...'.format(credentials_type))
people_service = None
if credentials_type == constants.API_KEY_CREDENTIALS_TYPE:
people_service = get_people_service_through_API_key()
elif credentials_type == constants.OAUTH_2_CREDENTIALS_TYPE:
people_service = get_people_service_through_OAuth_2()
return people_service

def get_profile_display_names_and_image_urls_with_getBatchGet(profile_ids,
credentials_type=constants.API_KEY_CREDENTIALS_TYPE,
verbose=False):
print('getting profile display names and image urls through {}...'.format(credentials_type))
display_names = []
image_urls = []

people_service = get_people_service(credentials_type=credentials_type)
people_resource = people_service.people()

BATCH_SIZE = 50
error_counter = 0
i = 0
while i < len(profile_ids):
    time.sleep(0.5)

    current_part_size = min(BATCH_SIZE, len(profile_ids) - i)
    current_profile_ids = profile_ids[i: i + current_part_size]
    resource_names = list(map(lambda string: 'people/' + string, current_profile_ids))

    # trying to get people document
    people_document = None
    delay = 0.5
    while people_document is None:
        time.sleep(delay)
        try:
            people_document = people_resource.getBatchGet(resourceNames=resource_names,
                                                          personFields='names,photos').execute()
        except apiclient.errors.HttpError:
            delay = 2
            print('{} | ERROR (apiclient.errors.HttpError [429])'.format(i))
            error_counter += 1
            continue

    # adding new data to display_names and image_urls
    new_data = {}
    for response in people_document['responses']:
        person = response.get('person', None)
        if person is None:
            print('Bad response: {}'.format(response))
            resource_name = response['requestedResourceName']
            person = {'resourceName': resource_name}

        resource_name = person['resourceName']  # 'people/114822392780451536009'
        profile_id = resource_name.split('/')[1]  # '114822392780451536009'
        # getting displayName
        if person.get('names', None) is None:
            print('{} | {} | Error. Names through People API is not available.'.format(profile_id,
                                                                                       plus_api.profile_id_to_google_plus_url(profile_id)))
            print('trying to get profile display name through Google+ API...')
            display_name = plus_api.get_profile_display_name(profile_id, credentials_type=credentials_type,
                                                             verbose=verbose)
            print('received display name: {}'.format(display_name))
            print('{} | {} | {}'.format(profile_id, plus_api.profile_id_to_google_plus_url(profile_id),
                                        display_name))
        else:
            display_name = person['names'][0]['displayName']
        # getting imageUrl
        if person.get('photos', None) is None:
            print('{} | {} | {} | Error. Photos through People API is not available.'.format(profile_id,
                                                                                             plus_api.profile_id_to_google_plus_url(profile_id), display_name))
            print('trying to get profile image url through Google+ API...')
            image_url = plus_api.get_profile_image_url(profile_id, credentials_type=credentials_type, verbose=verbose)
            print('received image url: {}'.format(image_url))
            print('{} | {} | {} | {}'.format(profile_id, plus_api.profile_id_to_google_plus_url(profile_id), display_name, image_url))
        else:
            image_url = person['photos'][0]['url']
        new_data[profile_id] = {'displayName': display_name, 'imageUrl': image_url}

    for profile_id in current_profile_ids:
        display_names.append(new_data[profile_id]['displayName'])
        image_urls.append(new_data[profile_id]['imageUrl'])

    i += current_part_size

    if verbose:
        print('{} | +{}'.format(i, current_part_size))

if error_counter != 0:
    print('{} errors (apiclient.errors.HttpError [429])'.format(error_counter))

# people_document = people_resource.getBatchGet(resourceNames=['people/114822392780451536009', 'people/106651989741536097256'], personFields='names,photos').execute()

return display_names, image_urls

def get_profile_display_names_and_image_urls(profile_ids, credentials_type=constants.API_KEY_CREDENTIALS_TYPE, verbose=False):
print('getting profile display names and image urls through {}...'.format(credentials_type))
display_names = []
image_urls = []

people_service = get_people_service(credentials_type=credentials_type)
people_resource = people_service.people()

error_counter = 0
for i, profile_id in enumerate(profile_ids):
    people_document = None
    delay = 0.5
    while people_document is None:
        time.sleep(delay)
        try:
            people_document = people_resource.get(resourceName='people/{}'.format(profile_id),
                                                  personFields='names,photos').execute()
        except apiclient.errors.HttpError:
            delay = 2
            print('{} | {} | ERROR'.format(i, profile_id))
            error_counter += 1
            continue
    display_name = people_document['names'][0]['displayName']
    display_names.append(display_name)
    image_url = people_document['photos'][0]['url']
    image_urls.append(image_url)
    if verbose:
        print('{} | {} | {} | {} | {}'.format(i, profile_id, plus_api.profile_id_to_google_plus_url(profile_id),
                                              display_name, image_url))
    # time.sleep(0.5)
print('{} errors'.format(error_counter))

return display_names, image_urls

def get_profiles(count, credentials_type=constants.API_KEY_CREDENTIALS_TYPE, verbose=False):
"""
[
{
profile_id,
profile_url,
display_name,
image_url
}
]
"""
print('getting profiles through {}...'.format(credentials_type))

# getting profiles information
profile_ids = plus_api.get_unique_profile_ids(count, credentials_type=credentials_type, verbose=verbose)
profile_urls = list(map(plus_api.profile_id_to_google_plus_url, profile_ids))
# display_names, image_urls = get_profile_display_names_and_image_urls(profile_ids, credentials_type=credentials_type,
#                                                                      verbose=verbose)
display_names, image_urls = get_profile_display_names_and_image_urls_with_getBatchGet(profile_ids,
                                                                                      credentials_type=credentials_type,
                                                                                      verbose=verbose)

# saving as plain text
txt_filename = 'results/people-api-profiles.txt'
if verbose:
    print('saving profiles as plain text to {}...'.format(txt_filename))
result_str = ''
for profile in zip(profile_ids, profile_urls, display_names, image_urls):
    result_str += '\n'.join(profile) + '\n\n'
profiles_file = open(txt_filename, 'w', encoding='utf-8')
profiles_file.write(result_str)
profiles_file.close()

# saving as json
json_filename = 'results/people-api-profiles.json'
if verbose:
    print('saving profiles as json to {}...'.format(json_filename))
result_list = []
json_fields = ('profileId', 'profileUrl', 'displayName', 'imageUrl')
for profile in zip(profile_ids, profile_urls, display_names, image_urls):
    result_list.append(dict(zip(json_fields, profile)))
json_str = json.dumps(result_list)
profiles_file = open(json_filename, 'w', encoding='utf-8')
profiles_file.write(json_str)
profiles_file.close()

if name == 'main':
start_time = time.time()
get_profiles(10000, verbose=True)
print('Total time: {} s'.format(round(time.time() - start_time, 2)))

# TODO get_profiles_through_generator
# TODO unique_profile_ids_generator OR pass new_seed (custom start query, not 'a') to query_generator

[christianheitkamp12-boop]

import apiclient
import oauth2client
import httplib2
import time
import random
import json

import GoogleAPI.constants as constants

import file_handler

def get_OAuth_2_credentials():
store = oauth2client.file.Storage(constants.CREDENTIAL_PATH)
credentials = store.get()
if not credentials or credentials.invalid:
flow = oauth2client.client.flow_from_clientsecrets(constants.CLIENT_SECRET_FILE, constants.SCOPES)
flow.user_agent = constants.APPLICATION_NAME
credentials = oauth2client.tools.run_flow(flow, store)
print('Storing credentials to ' + constants.CREDENTIAL_PATH)
return credentials

def get_plus_service_through_API_key():
return apiclient.discovery.build('plus', 'v1', developerKey=constants.API_KEY)

def get_plus_service_through_OAuth_2():
credentials = get_OAuth_2_credentials()
http = credentials.authorize(httplib2.Http())
return apiclient.discovery.build('plus', 'v1', http=http)
# ,discoveryServiceUrl='https://people.googleapis.com/$discovery/rest')

def get_plus_service(credentials_type='API key'):
print('getting Google+ service through {}...'.format(credentials_type))
plus_service = None
if credentials_type == 'API key':
plus_service = get_plus_service_through_API_key()
elif credentials_type == 'OAuth 2.0':
plus_service = get_plus_service_through_OAuth_2()
return plus_service

def profile_id_to_google_plus_url(profile_id):
return 'https://plus.google.com/' + profile_id

def get_profile_data_as_json(profile_id, credentials_type='API key'):
print('getting data of https://plus.google.com/u/0/{} as json through {}...'.format(profile_id, credentials_type))
plus_service = get_plus_service(credentials_type=credentials_type)
return plus_service.people().get(userId=profile_id).execute()

def query_generator():
russian_alphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
english_alphabet = 'abcdefghijklmnopqrstuvwxyz'
digits = '0123456789'
for char in english_alphabet:
yield char
for char in russian_alphabet:
yield char
for char in digits:
yield char
for first_char in english_alphabet:
for second_char in english_alphabet:
yield first_char + second_char
for first_char in russian_alphabet:
for second_char in russian_alphabet:
yield first_char + second_char

yield str(random.randint(0, 10000000))

def get_unique_profile_ids(count=10, credentials_type='API key', verbose=False):
print('searching for {} unique Google profile IDs...'.format(count))

unique_profile_ids = set()

plus_service = get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()
# max_results = 10

query_gen = query_generator()
query = next(query_gen)
next_page_token = ''
current_count = 0
query_count = 0
MAX_QUERY_COUNT = 25000
while current_count < count and query_count < MAX_QUERY_COUNT:
    max_results = min(50, count - current_count)
    people_document = people_resource.search(query=query, maxResults=max_results,
                                             pageToken=next_page_token).execute()
    query_count += 1
    ids = [item['id'] for item in people_document['items']]
    next_page_token = people_document['nextPageToken']

    # if len(ids) < 10:
    if next_page_token == '':
        query = next(query_gen)

    prev_len = len(unique_profile_ids)
    unique_profile_ids.update(ids)
    current_count += len(unique_profile_ids) - prev_len

    time.sleep(1)

    if verbose:
        print('{} | +{} | query: {} | nextPageToken: {}'.format(current_count, current_count - prev_len, query,
                                                                next_page_token))

return list(unique_profile_ids)

def get_profile_display_name(profile_id, credentials_type='API key', verbose=False):
print('getting profile display name through {}...'.format(credentials_type))
plus_service = get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()
people_document = people_resource.get(userId=profile_id).execute()
display_name = people_document['displayName']
return display_name

def get_profile_image_url(profile_id, credentials_type='API key', verbose=False):
print('getting profile image url through {}...'.format(credentials_type))
plus_service = get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()
people_document = people_resource.get(userId=profile_id).execute()
image_url = people_document['image']['url']
return image_url

def get_profile_image_urls(profile_ids, credentials_type='API key', verbose=False):
print('getting profile image urls through {}...'.format(credentials_type))
profile_image_urls = []

plus_service = get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()
for i, profile_id in enumerate(profile_ids):
    people_document = people_resource.get(userId=profile_id).execute()
    image_url = people_document['image']['url']
    profile_image_urls.append(image_url)
    if verbose:
        print('{} | {} | {} | {}'.format(i, profile_id, profile_id_to_google_plus_url(profile_id), image_url))

return profile_image_urls

def get_profile_display_names(profile_ids, credentials_type='API key', verbose=False):
print('getting profile display names through {}...'.format(credentials_type))
display_names = []

plus_service = get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()
for i, profile_id in enumerate(profile_ids):
    people_document = people_resource.get(userId=profile_id).execute()
    display_name = people_document['displayName']
    display_names.append(display_name)
    if verbose:
        print('{} | {} | {} | {}'.format(i, profile_id, profile_id_to_google_plus_url(profile_id), display_name))

return display_names

def get_profiles(count, credentials_type='API key', verbose=False):
"""
[
{
profile_id,
profile_url,
display_name,
image_url
}
]
"""
print('getting profiles through {}...'.format(credentials_type))

# getting profiles information
profile_ids = get_unique_profile_ids(count, credentials_type=credentials_type, verbose=verbose)
profile_urls = list(map(profile_id_to_google_plus_url, profile_ids))
display_names = get_profile_display_names(profile_ids, credentials_type=credentials_type, verbose=verbose)
image_urls = get_profile_image_urls(profile_ids, credentials_type=credentials_type, verbose=verbose)

# saving as plain text
txt_filename = 'results/plus-api-profiles.txt'
if verbose:
    print('saving profiles as plain text to {}...'.format(txt_filename))
result_str = ''
for profile in zip(profile_ids, profile_urls, display_names, image_urls):
    result_str += '\n'.join(profile) + '\n\n'
profiles_file = open(txt_filename, 'w', encoding='utf-8')
profiles_file.write(result_str)
profiles_file.close()

# saving as json
json_filename = 'results/plus-api-profiles.json'
if verbose:
    print('saving profiles as json to {}...'.format(json_filename))
result_list = []
json_fields = ('profileId', 'profileUrl', 'displayName', 'imageUrl')
for profile in zip(profile_ids, profile_urls, display_names, image_urls):
    result_list.append(dict(zip(json_fields, profile)))
json_str = json.dumps(result_list)
profiles_file = open(json_filename, 'w', encoding='utf-8')
profiles_file.write(json_str)
profiles_file.close()

if name == 'main':
get_profiles(10, verbose=True, credentials_type='OAuth 2.0')

# unique_profile_ids = get_unique_profile_ids(2000, verbose=True)
# # print('{} ids'.format(len(unique_profile_ids)))
# file_handler.write_list_to('unique-profile-ids.txt', unique_profile_ids)
# # print_urls(list(map(profile_id_to_google_plus_url, unique_profile_ids)))
#
# ids = file_handler.read_list_from('unique-profile-ids.txt')
# profile_ids = ids[:100]
# display_names = get_profile_display_names(profile_ids, verbose=True)
# file_handler.write_list_to('unique-profile-display-names.txt', display_names)


# profile_image_urls = get_profile_image_urls(profile_ids, verbose=True)
# profile_urls = list(map(profile_id_to_google_plus_url, profile_ids))
# file_handler.write_list_to('unique-profile-image-urls.txt', profile_image_urls)
# for profile_url, image_url in zip(profile_urls, profile_image_urls):
#     print('{} {}'.format(profile_url, image_url))
# print(len(ids))
# print(ids[:10])
# unique_profile_urls = list(map(profile_id_to_google_plus_url, ids))
# file_handler.write_list_to('unique-profile-urls.txt', unique_profile_urls)

[christianheitkamp12-boop]

import apiclient
import oauth2client
import httplib2
import time
import random
import json

import GoogleAPI.constants as constants

import file_handler

def get_OAuth_2_credentials():
store = oauth2client.file.Storage(constants.CREDENTIAL_PATH)
credentials = store.get()
if not credentials or credentials.invalid:
flow = oauth2client.client.flow_from_clientsecrets(constants.CLIENT_SECRET_FILE, constants.SCOPES)
flow.user_agent = constants.APPLICATION_NAME
credentials = oauth2client.tools.run_flow(flow, store)
print('Storing credentials to ' + constants.CREDENTIAL_PATH)
return credentials

def get_plus_service_through_API_key():
return apiclient.discovery.build('plus', 'v1', developerKey=constants.API_KEY)

def get_plus_service_through_OAuth_2():
credentials = get_OAuth_2_credentials()
http = credentials.authorize(httplib2.Http())
return apiclient.discovery.build('plus', 'v1', http=http)
# ,discoveryServiceUrl='https://people.googleapis.com/$discovery/rest')

def get_plus_service(credentials_type='API key'):
print('getting Google+ service through {}...'.format(credentials_type))
plus_service = None
if credentials_type == 'API key':
plus_service = get_plus_service_through_API_key()
elif credentials_type == 'OAuth 2.0':
plus_service = get_plus_service_through_OAuth_2()
return plus_service

def profile_id_to_google_plus_url(profile_id):
return 'https://plus.google.com/' + profile_id

def get_profile_data_as_json(profile_id, credentials_type='API key'):
print('getting data of https://plus.google.com/u/0/{} as json through {}...'.format(profile_id, credentials_type))
plus_service = get_plus_service(credentials_type=credentials_type)
return plus_service.people().get(userId=profile_id).execute()

def query_generator():
russian_alphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
english_alphabet = 'abcdefghijklmnopqrstuvwxyz'
digits = '0123456789'
for char in english_alphabet:
yield char
for char in russian_alphabet:
yield char
for char in digits:
yield char
for first_char in english_alphabet:
for second_char in english_alphabet:
yield first_char + second_char
for first_char in russian_alphabet:
for second_char in russian_alphabet:
yield first_char + second_char

yield str(random.randint(0, 10000000))

def get_unique_profile_ids(count=10, credentials_type='API key', verbose=False):
print('searching for {} unique Google profile IDs...'.format(count))

unique_profile_ids = set()

plus_service = get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()
# max_results = 10

query_gen = query_generator()
query = next(query_gen)
next_page_token = ''
current_count = 0
query_count = 0
MAX_QUERY_COUNT = 25000
while current_count < count and query_count < MAX_QUERY_COUNT:
    max_results = min(50, count - current_count)
    people_document = people_resource.search(query=query, maxResults=max_results,
                                             pageToken=next_page_token).execute()
    query_count += 1
    ids = [item['id'] for item in people_document['items']]
    next_page_token = people_document['nextPageToken']

    # if len(ids) < 10:
    if next_page_token == '':
        query = next(query_gen)

    prev_len = len(unique_profile_ids)
    unique_profile_ids.update(ids)
    current_count += len(unique_profile_ids) - prev_len

    time.sleep(1)

    if verbose:
        print('{} | +{} | query: {} | nextPageToken: {}'.format(current_count, current_count - prev_len, query,
                                                                next_page_token))

return list(unique_profile_ids)

def get_profile_display_name(profile_id, credentials_type='API key', verbose=False):
print('getting profile display name through {}...'.format(credentials_type))
plus_service = get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()
people_document = people_resource.get(userId=profile_id).execute()
display_name = people_document['displayName']
return display_name

def get_profile_image_url(profile_id, credentials_type='API key', verbose=False):
print('getting profile image url through {}...'.format(credentials_type))
plus_service = get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()
people_document = people_resource.get(userId=profile_id).execute()
image_url = people_document['image']['url']
return image_url

def get_profile_image_urls(profile_ids, credentials_type='API key', verbose=False):
print('getting profile image urls through {}...'.format(credentials_type))
profile_image_urls = []

plus_service = get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()
for i, profile_id in enumerate(profile_ids):
    people_document = people_resource.get(userId=profile_id).execute()
    image_url = people_document['image']['url']
    profile_image_urls.append(image_url)
    if verbose:
        print('{} | {} | {} | {}'.format(i, profile_id, profile_id_to_google_plus_url(profile_id), image_url))

return profile_image_urls

def get_profile_display_names(profile_ids, credentials_type='API key', verbose=False):
print('getting profile display names through {}...'.format(credentials_type))
display_names = []

plus_service = get_plus_service(credentials_type=credentials_type)
people_resource = plus_service.people()
for i, profile_id in enumerate(profile_ids):
    people_document = people_resource.get(userId=profile_id).execute()
    display_name = people_document['displayName']
    display_names.append(display_name)
    if verbose:
        print('{} | {} | {} | {}'.format(i, profile_id, profile_id_to_google_plus_url(profile_id), display_name))

return display_names

def get_profiles(count, credentials_type='API key', verbose=False):
"""
[
{
profile_id,
profile_url,
display_name,
image_url
}
]
"""
print('getting profiles through {}...'.format(credentials_type))

# getting profiles information
profile_ids = get_unique_profile_ids(count, credentials_type=credentials_type, verbose=verbose)
profile_urls = list(map(profile_id_to_google_plus_url, profile_ids))
display_names = get_profile_display_names(profile_ids, credentials_type=credentials_type, verbose=verbose)
image_urls = get_profile_image_urls(profile_ids, credentials_type=credentials_type, verbose=verbose)

# saving as plain text
txt_filename = 'results/plus-api-profiles.txt'
if verbose:
    print('saving profiles as plain text to {}...'.format(txt_filename))
result_str = ''
for profile in zip(profile_ids, profile_urls, display_names, image_urls):
    result_str += '\n'.join(profile) + '\n\n'
profiles_file = open(txt_filename, 'w', encoding='utf-8')
profiles_file.write(result_str)
profiles_file.close()

# saving as json
json_filename = 'results/plus-api-profiles.json'
if verbose:
    print('saving profiles as json to {}...'.format(json_filename))
result_list = []
json_fields = ('profileId', 'profileUrl', 'displayName', 'imageUrl')
for profile in zip(profile_ids, profile_urls, display_names, image_urls):
    result_list.append(dict(zip(json_fields, profile)))
json_str = json.dumps(result_list)
profiles_file = open(json_filename, 'w', encoding='utf-8')
profiles_file.write(json_str)
profiles_file.close()

if name == 'main':
get_profiles(10, verbose=True, credentials_type='OAuth 2.0')

# unique_profile_ids = get_unique_profile_ids(2000, verbose=True)
# # print('{} ids'.format(len(unique_profile_ids)))
# file_handler.write_list_to('unique-profile-ids.txt', unique_profile_ids)
# # print_urls(list(map(profile_id_to_google_plus_url, unique_profile_ids)))
#
# ids = file_handler.read_list_from('unique-profile-ids.txt')
# profile_ids = ids[:100]
# display_names = get_profile_display_names(profile_ids, verbose=True)
# file_handler.write_list_to('unique-profile-display-names.txt', display_names)


# profile_image_urls = get_profile_image_urls(profile_ids, verbose=True)
# profile_urls = list(map(profile_id_to_google_plus_url, profile_ids))
# file_handler.write_list_to('unique-profile-image-urls.txt', profile_image_urls)
# for profile_url, image_url in zip(profile_urls, profile_image_urls):
#     print('{} {}'.format(profile_url, image_url))
# print(len(ids))
# print(ids[:10])
# unique_profile_urls = list(map(profile_id_to_google_plus_url, ids))
# file_handler.write_list_to('unique-profile-urls.txt', unique_profile_urls)

[christianheitkamp12-boop]

ImageMining

Image mining. Mine profile images through Google API (Google+ API and Google People API).

Dependencies

• python 3
• requests
• google-api-python-client
• project on Google API and credentials (API key or OAuth 2.0)

Preparations

1. Create folder GoogleAPI/resources
2. Add to GoogleAPI/resources/ your credentials files
   (api_key.txt (must contain only your API key) and client_secret.json (you will get it on credentials))
   (see constants.py to change default folder).

[christianheitkamp12-boop]

def write_list_to(filename, ls):
print('writing list to {}...'.format(filename))
file = open(filename, 'w', encoding='utf-8')
for item in ls:
file.write(item + '\n')
file.close()

def read_list_from(filename):
print('reading list from {}...'.format(filename))
file = open(filename, 'r', encoding='utf-8')
file_str = file.read()
file_str = file_str.rstrip()
return file_str.split('\n')

if name == 'main':
s = 'Bảo Trân kla'
# s = s[:2]
file = open('temp.txt', 'w', encoding='utf-8')
file.write(s + '\n')
file.close()
file = open('temp.txt', 'r', encoding='utf-8')
file_str = file.read()
file_str = file_str.rstrip()
print(file_str)

[christianheitkamp12-boop]

import apiclient
import oauth2client
import httplib2
import time
import random
import json

import GoogleAPI.constants as constants
import GoogleAPI.file_handler as file_handler

import GoogleAPI.plus_api

from plus_api import get_unique_profile_ids, profile_id_to_google_plus_url

def get_OAuth_2_credentials():
store = oauth2client.file.Storage(constants.OAUTH_2_CREDENTIALS_PATH)
credentials = store.get()
if not credentials or credentials.invalid:
flow = oauth2client.client.flow_from_clientsecrets(constants.CLIENT_SECRET_PATH, constants.SCOPES)
flow.user_agent = constants.APPLICATION_NAME
credentials = oauth2client.tools.run_flow(flow, store)
print('Storing credentials to ' + constants.OAUTH_2_CREDENTIALS_PATH)
return credentials

def get_people_service_through_API_key():
api_key_file = open(constants.API_KEY_CREDENTIALS_PATH, 'r', encoding='utf-8')
api_key = api_key_file.read().strip()
return apiclient.discovery.build('people', 'v1', developerKey=api_key)

def get_people_service_through_OAuth_2():
credentials = get_OAuth_2_credentials()
http = credentials.authorize(httplib2.Http())
return apiclient.discovery.build('people', 'v1', http=http)
# ,discoveryServiceUrl='https://people.googleapis.com/$discovery/rest')

def get_people_service(credentials_type=constants.API_KEY_CREDENTIALS_TYPE):
print('getting People API service through {}...'.format(credentials_type))
people_service = None
if credentials_type == constants.API_KEY_CREDENTIALS_TYPE:
people_service = get_people_service_through_API_key()
elif credentials_type == constants.OAUTH_2_CREDENTIALS_TYPE:
people_service = get_people_service_through_OAuth_2()
return people_service

def get_profile_display_names_and_image_urls_with_getBatchGet(profile_ids,
credentials_type=constants.API_KEY_CREDENTIALS_TYPE,
verbose=False):
print('getting profile display names and image urls through {}...'.format(credentials_type))
display_names = []
image_urls = []

people_service = get_people_service(credentials_type=credentials_type)
people_resource = people_service.people()

BATCH_SIZE = 50
error_counter = 0
i = 0
while i < len(profile_ids):
    time.sleep(0.5)

    current_part_size = min(BATCH_SIZE, len(profile_ids) - i)
    current_profile_ids = profile_ids[i: i + current_part_size]
    resource_names = list(map(lambda string: 'people/' + string, current_profile_ids))

    # trying to get people document
    people_document = None
    delay = 0.5
    while people_document is None:
        time.sleep(delay)
        try:
            people_document = people_resource.getBatchGet(resourceNames=resource_names,
                                                          personFields='names,photos').execute()
        except apiclient.errors.HttpError:
            delay = 2
            print('{} | ERROR (apiclient.errors.HttpError [429])'.format(i))
            error_counter += 1
            continue

    # adding new data to display_names and image_urls
    new_data = {}
    for response in people_document['responses']:
        person = response.get('person', None)
        if person is None:
            print('Bad response: {}'.format(response))
            resource_name = response['requestedResourceName']
            person = {'resourceName': resource_name}

        resource_name = person['resourceName']  # 'people/114822392780451536009'
        profile_id = resource_name.split('/')[1]  # '114822392780451536009'
        # getting displayName
        if person.get('names', None) is None:
            print('{} | {} | Error. Names through People API is not available.'.format(profile_id,
                                                                                       plus_api.profile_id_to_google_plus_url(profile_id)))
            print('trying to get profile display name through Google+ API...')
            display_name = plus_api.get_profile_display_name(profile_id, credentials_type=credentials_type,
                                                             verbose=verbose)
            print('received display name: {}'.format(display_name))
            print('{} | {} | {}'.format(profile_id, plus_api.profile_id_to_google_plus_url(profile_id),
                                        display_name))
        else:
            display_name = person['names'][0]['displayName']
        # getting imageUrl
        if person.get('photos', None) is None:
            print('{} | {} | {} | Error. Photos through People API is not available.'.format(profile_id,
                                                                                             plus_api.profile_id_to_google_plus_url(profile_id), display_name))
            print('trying to get profile image url through Google+ API...')
            image_url = plus_api.get_profile_image_url(profile_id, credentials_type=credentials_type, verbose=verbose)
            print('received image url: {}'.format(image_url))
            print('{} | {} | {} | {}'.format(profile_id, plus_api.profile_id_to_google_plus_url(profile_id), display_name, image_url))
        else:
            image_url = person['photos'][0]['url']
        new_data[profile_id] = {'displayName': display_name, 'imageUrl': image_url}

    for profile_id in current_profile_ids:
        display_names.append(new_data[profile_id]['displayName'])
        image_urls.append(new_data[profile_id]['imageUrl'])

    i += current_part_size

    if verbose:
        print('{} | +{}'.format(i, current_part_size))

if error_counter != 0:
    print('{} errors (apiclient.errors.HttpError [429])'.format(error_counter))

# people_document = people_resource.getBatchGet(resourceNames=['people/114822392780451536009', 'people/106651989741536097256'], personFields='names,photos').execute()

return display_names, image_urls

def get_profile_display_names_and_image_urls(profile_ids, credentials_type=constants.API_KEY_CREDENTIALS_TYPE, verbose=False):
print('getting profile display names and image urls through {}...'.format(credentials_type))
display_names = []
image_urls = []

people_service = get_people_service(credentials_type=credentials_type)
people_resource = people_service.people()

error_counter = 0
for i, profile_id in enumerate(profile_ids):
    people_document = None
    delay = 0.5
    while people_document is None:
        time.sleep(delay)
        try:
            people_document = people_resource.get(resourceName='people/{}'.format(profile_id),
                                                  personFields='names,photos').execute()
        except apiclient.errors.HttpError:
            delay = 2
            print('{} | {} | ERROR'.format(i, profile_id))
            error_counter += 1
            continue
    display_name = people_document['names'][0]['displayName']
    display_names.append(display_name)
    image_url = people_document['photos'][0]['url']
    image_urls.append(image_url)
    if verbose:
        print('{} | {} | {} | {} | {}'.format(i, profile_id, plus_api.profile_id_to_google_plus_url(profile_id),
                                              display_name, image_url))
    # time.sleep(0.5)
print('{} errors'.format(error_counter))

return display_names, image_urls

def get_profiles(count, credentials_type=constants.API_KEY_CREDENTIALS_TYPE, verbose=False):
"""
[
{
profile_id,
profile_url,
display_name,
image_url
}
]
"""
print('getting profiles through {}...'.format(credentials_type))

# getting profiles information
profile_ids = plus_api.get_unique_profile_ids(count, credentials_type=credentials_type, verbose=verbose)
profile_urls = list(map(plus_api.profile_id_to_google_plus_url, profile_ids))
# display_names, image_urls = get_profile_display_names_and_image_urls(profile_ids, credentials_type=credentials_type,
#                                                                      verbose=verbose)
display_names, image_urls = get_profile_display_names_and_image_urls_with_getBatchGet(profile_ids,
                                                                                      credentials_type=credentials_type,
                                                                                      verbose=verbose)

# saving as plain text
txt_filename = 'results/people-api-profiles.txt'
if verbose:
    print('saving profiles as plain text to {}...'.format(txt_filename))
result_str = ''
for profile in zip(profile_ids, profile_urls, display_names, image_urls):
    result_str += '\n'.join(profile) + '\n\n'
profiles_file = open(txt_filename, 'w', encoding='utf-8')
profiles_file.write(result_str)
profiles_file.close()

# saving as json
json_filename = 'results/people-api-profiles.json'
if verbose:
    print('saving profiles as json to {}...'.format(json_filename))
result_list = []
json_fields = ('profileId', 'profileUrl', 'displayName', 'imageUrl')
for profile in zip(profile_ids, profile_urls, display_names, image_urls):
    result_list.append(dict(zip(json_fields, profile)))
json_str = json.dumps(result_list)
profiles_file = open(json_filename, 'w', encoding='utf-8')
profiles_file.write(json_str)
profiles_file.close()

if name == 'main':
start_time = time.time()
get_profiles(10000, verbose=True)
print('Total time: {} s'.format(round(time.time() - start_time, 2)))

# TODO get_profiles_through_generator
# TODO unique_profile_ids_generator OR pass new_seed (custom start query, not 'a') to query_generator

[christianheitkamp12-boop]

API key

API_KEY_CREDENTIALS_PATH = 'resources/api_key.txt'

OAuth 2.0

SCOPES = [
'https://www.googleapis.com/auth/contacts',
'https://www.googleapis.com/auth/contacts.readonly',
'https://www.googleapis.com/auth/plus.login',
'https://www.googleapis.com/auth/user.addresses.read',
'https://www.googleapis.com/auth/user.birthday.read',
'https://www.googleapis.com/auth/user.emails.read',
'https://www.googleapis.com/auth/user.phonenumbers.read',
'email',
'https://www.googleapis.com/auth/plus.profile.emails.read',
'profile',
]
CLIENT_SECRET_PATH = 'resources/client_secret.json'
APPLICATION_NAME = 'People API Parser'
OAUTH_2_CREDENTIALS_PATH = 'resources/people-and-plus.googleapis.com-credentials.json'

Credentials types

API_KEY_CREDENTIALS_TYPE = 'API key'
OAUTH_2_CREDENTIALS_TYPE = 'OAuth 2.0'

Others

EMPTY_BINARY_STRING = b''