Notes

import copy
import math
from MindsiteCrawling.data.url_queue import UrlQueue
# from MindsiteCrawling.dbaccess.evkiba_oos_masterdata_input_for_crawling import get_links_from_evkiba_masterdata
# from MindsiteCrawling.dbaccess.external_url_input import get_external_links
from MindsiteCrawling.dbaccess.crawling_input import get_price_product_urls
from MindsiteCrawling.spider.spider import Spider
from MindsiteCrawling.data.crawling_meta_model import ParsedData, CrawlingMeta
import datetime

# from masterdata_crawling_input import get_all_links_from_masterdatas

from MindsiteCrawling.dbaccess.crawling_input import get_price_product_urls, get_content_product_urls

class N11Spider(Spider):
def __init__(self):
self.name = 'N11'
self.base_url = 'https://mobileapi.n11.com'
# self.product_per_page = 28
# self.max_product_page = 50
# self.page_extension = '&pg='
# self.min_price_extension = '&minp='
# self.max_price_extension = '&maxp='
#
self.has_other_merchant = False
self.other_merchant_new_request = False
self.has_option = True
self.option_new_request = False
#
self.proxy_status = True
self.proxy_type = 'data_center'
self.cookie_status = False
self.page_size = 18
#
self.header = {
'Authorization': 'api_key=android,api_hash=b9ae9cd60fe1179e70ff24a61b18a57c,api_random=5119515692952717060', # category api key
#api_key=android,api_hash=9292cf5dcacc965ef5b502007ba6558a,api_random=-130315708053012896 #product api key
'MobileClient': 'ANDROID',
'ClientVersion': '156',
'Host': 'mobileapi.n11.com',
'Connection': 'Keep-Alive',
'Accept-Encoding': 'gzip',
'User-Agent': 'okhttp/3.9.1',
'ADRUM_1': 'isMobile:true',
'ADRUM': 'isAjax:true'
}

def get_callback_function(self):
return self.callback_functions.pop(0)


class N11PriceSpider(N11Spider):
def __init__(self, delay):
super().__init__()
self.delay = delay
self.min_content_length = 0

self.callback_functions = [
self.generate_category_pages,
self.generate_sku_urls,
self.generate_sku_data
]
self.product_count = {
'json': {
1: [{'key': 'pagination'}, {'key': 'totalCount'}]
},
'process': lambda meta, unp_data, source, no: int(unp_data)
}
self.skus = {
'json': {
1: [{'key': 'productListingItems'}]
},
'process': lambda meta, unp_data, source, no: unp_data
}
self.product_id = {
'json': {
1: [{'key': 'id'}]
},
'process': lambda meta, unp_data, source, no: str(unp_data)
}
self.product_title = {
'json': {
1: [{'key': 'product'}, {'key': 'title'}]
},
'process': lambda meta, unp_data, source, no: unp_data
}
self.product_url = {
'json': {
1: [{'key': 'product'}, {'key': 'productUrl'}]
},
'process': lambda meta, unp_data, source, no: unp_data
}
self.product_image_url = {
'json': {
1: [{'key': 'product'}, {'key': 'images'}]
},
'process': self.parse_product_image_url
}
self.product_barcode = {
'json': {
1: [{'key': 'product'}, {'key': 'defaultGtin'}]
},
'process': lambda meta, unp_data, source, no: unp_data
}
self.product_subtitle = {
'json': {
1: [{'key': 'product'}, {'key': 'subTitle'}]
},
'process': lambda meta, unp_data, source, no: unp_data
}
self.merchant_name = {
'json': {
1: [{'key': 'product'}, {'key': 'seller'}, {'key': 'nickName'}]
},
'process': lambda meta, unp_data, source, no: unp_data
}
self.product_price = {
'json': {
1: [{'key': 'product'}, {'key': 'displayPriceNumber'}]
},
'process': lambda meta, unp_data, source, no: unp_data
}
self.product_price_2 = {
'default_value': None
}
self.cargo_fee = {
'json': {
1: [{'key': 'product'}, {'key': 'shippingFree'}]
},
'process': lambda meta, unp_data, source, no: 0.0 if bool(unp_data) else 10.90
}
self.product_stock = {
'json': {
1: [{'key': 'product'}, {'key': 'stock'}]
},
'process': lambda meta, unp_data, source, no: unp_data
}
self.group_id = {
'json': {
1: [{'key': 'product'}, {'key': 'groupId'}]
},
'process': lambda meta, unp_data, source, no: str(unp_data)
}
self.product_availability = {
'json': {
1: [{'key': 'product'}, {'key': 'stock'}]
},
'process': lambda meta, unp_data, source, no: False if int(unp_data) == 0 else True
}
self.brand = {
'json': {
1: [{'key': 'product'}, {'key': 'productBrand'}]},
'process': lambda meta, unp_data, source, no: unp_data
}
self.options = {
'json': {
1: [{'key': 'product'}, {'key': 'skus'}]},
'process': lambda meta, unp_data, source, no: unp_data,
'default_value': []
}
self.option_id = {
'json': {
1: [{'key': 'id'}]},
'process': lambda meta, unp_data, source, no: str(unp_data)
}
self.option_title = {
'json': {
1: [{'key': 'skuAttributes'}, {'index': 0}, {'key': 'name'}]},
'process': lambda meta, unp_data, source, no: unp_data
}
self.option_barcode = {
'json': {
1: [{'key': 'gtin'}]},
'process': lambda meta, unp_data, source, no: str(unp_data)
}
self.option_stock = {
'json': {
1: [{'key': 'currentStock'}]},
'process': lambda meta, unp_data, source, no: unp_data
}
self.option_price = {
'json': {
1: [{'key': 'displayPriceNumber'}]},
'process': self.parse_option_price
}
self.product_barcode = {
'json': {
1: [{'key': 'product'}, {'key': 'defaultGtin'}]},
'process': lambda meta, unp_data, source, no: str(unp_data)
}

@staticmethod
def parse_option_price(meta, unp_data, source, no):
if unp_data == 'None':
return meta.data.product_price
else:
return unp_data

@staticmethod
def parse_product_image_url(meta, unp_data, source, no):
try:
return unp_data[0]['path'].replace('{0}', '450')
except:
return 'NO_URL'

def generate_input_metas(self, start_meta: CrawlingMeta, queue: UrlQueue):
def set_input_settings(spider, input):
input['has_other_merchant'] = spider.has_other_merchant
input['other_merchant_new_request'] = spider.other_merchant_new_request
input['has_option'] = spider.has_option
input['option_new_request'] = spider.option_new_request
return input

new_input = {key: value for key, value in start_meta.start_data.items() if
key not in ['crawling_input', 'category_ids']}
for no, db_input in enumerate(start_meta.start_data['crawling_input']):
spider = start_meta.start_data['spiders_dict'][db_input['retailer']][db_input['category_type']]
new_input = set_input_settings(spider, new_input)
new_input['source_url'] = db_input['category_url']
new_input['brand'] = db_input['brand']
new_input['category'] = db_input['category']
new_input['subcategory'] = db_input['subcategory']
url = db_input['category_url'] + '&page=1'
print(no, url)
new_spider = copy.deepcopy(spider)
new_spider.callback_functions = copy.deepcopy(self.callback_functions)
meta = CrawlingMeta(initial_data=new_input,
url=url,
spider=new_spider,
request_method='get',
request_type='json')
queue.add_url(meta)

def generate_category_pages(self, meta, queue, parse_field):
parse_field(meta, queue, 'product_count')
print('PRODUCT_COUNT::', meta.data.product_count)
product_count = meta.data.product_count
setattr(meta.initial_data, 'product_count', product_count)

for i in range(math.ceil(product_count / self.page_size)):
url = meta.url.split('&page=')[0] + '&page=' + str(i + 1)
new_spider = copy.deepcopy(meta.spider)
new_spider.callback_functions = copy.deepcopy(self.callback_functions)

queue.add_url(CrawlingMeta(initial_data=meta.initial_data.__dict__,
url=url,
spider=new_spider,
request_method='get',
request_type='json'))

def generate_sku_urls(self, meta: CrawlingMeta, queue, parse_field):
# print('SKU_JSON::', meta.json)
parse_field(meta, queue, 'skus')
# print('SKUS::', meta.data.skus)
# print('DATA_DETAIL', meta.data_detail)
# print('JSON_DETAIL', meta.json_detail)
# print('PAGE', meta.url)

for sku in meta.data.skus:
p_meta = CrawlingMeta(initial_data=copy.deepcopy(meta.initial_data.__dict__),
request_method='get',
request_type='json',
json='sku')
new_spider: N11Spider = copy.deepcopy(meta.spider)
new_spider.callback_functions = copy.deepcopy(self.callback_functions)
p_meta.json = sku
p_meta.spider = new_spider
parse_field(p_meta, queue, 'product_id')
# if p_meta.data.product_id in ['407993369', '368724395']:
print('PRODUCT_ID:', p_meta.data.product_id)
p_meta.url = 'https://mobileapi.n11.com/mobileapi/rest/getProductNonPersonalizedDetail?productId={}&vehicleType=&subChannel=N11&productDetailType=N11&adBidding=false'.format(p_meta.data.product_id)
queue.add_url(p_meta)

def generate_sku_data(self, meta, queue, parse_field):
parse_field(meta, queue, 'options')
parse_field(meta, queue, 'product_url')
parse_field(meta, queue, 'product_image_url')
parse_field(meta, queue, 'merchant_name')
parse_field(meta, queue, 'brand')
parse_field(meta, queue, 'product_title')
parse_field(meta, queue, 'product_id')
parse_field(meta, queue, 'product_subtitle')
parse_field(meta, queue, 'group_id')
print('GROUP_ID:', meta.data.group_id)

if len(meta.data.options) > 1:
self.generate_option_data(meta, queue, parse_field)
return

parse_field(meta, queue, 'product_barcode')
setattr(meta.data, 'product_price_2', None)
setattr(meta.data, 'promotion_text', None)
parse_field(meta, queue, 'product_availability')

if meta.data.product_availability:
setattr(meta.data, 'is_buybox', True)
setattr(meta.data, 'buybox_no', 1)
parse_field(meta, queue, 'product_price')
parse_field(meta, queue, 'cargo_fee')
parse_field(meta, queue, 'product_stock')

else:
setattr(meta.data, 'is_buybox', False)
setattr(meta.data, 'buybox_no', None)
setattr(meta.data, 'product_price', None)
setattr(meta.data, 'cargo_fee', None)
setattr(meta.data, 'product_stock', 0)
self.log_fields(meta, 'ALL')
self.add_sku_data(meta.initial_data, meta.data, queue)

def generate_option_data(self, meta, queue, parse_field):

for option in meta.data.options:
parse_field(meta, queue, 'cargo_fee')
parse_field(meta, queue, 'product_availability')
parse_field(meta, queue, 'product_price')
option_meta = copy.deepcopy(meta)
option_meta.json = copy.deepcopy(option)

parse_field(option_meta, queue, 'option_id')
setattr(option_meta.data, 'product_id', option_meta.data.product_id + '__' + option_meta.data.option_id)

parse_field(option_meta, queue, 'option_title')
setattr(option_meta.data, 'product_title', option_meta.data.product_title + '__' + option_meta.data.option_title)

parse_field(option_meta, queue, 'option_barcode')
setattr(option_meta.data, 'product_barcode', option_meta.data.option_barcode)

setattr(option_meta.data, 'product_price_2', None)
setattr(option_meta.data, 'promotion_text', None)

# print('OPTION_DATA', option_meta.data)

if option_meta.data.product_availability:
setattr(option_meta.data, 'is_buybox', True)
setattr(option_meta.data, 'buybox_no', 1)
parse_field(option_meta, queue, 'option_price')
if option_meta.data.option_price is not None:
setattr(option_meta.data, 'product_price', option_meta.data.option_price)
parse_field(option_meta, queue, 'product_stock')

else:
setattr(option_meta.data, 'is_buybox', False)
setattr(option_meta.data, 'buybox_no', None)
setattr(option_meta.data, 'product_price', None)
setattr(option_meta.data, 'cargo_fee', None)
setattr(option_meta.data, 'product_stock', 0)

self.add_sku_data(option_meta.initial_data, option_meta.data, queue)


class N11OOSBuyboxSpider(N11PriceSpider):
def __init__(self, delay):
super().__init__(delay)
self.delay = delay

self.callback_functions = [
self.generate_sku_data
]
self.product_id = {
'json': {
1: [{'key': 'product'}, {'key': 'id'}]
},
'process': lambda meta, unp_data, source, no: str(unp_data)
}

def generate_input_metas(self, input, spider, crawling_input_queue, db):
# crawling_input_queue.url_masterdata, a = get_all_links_from_masterdatas()
urls = crawling_input_queue.url_masterdata.get(self.name, [])

# crawling_input_queue.url_external = get_external_links(db)
external_urls = crawling_input_queue.url_external.get(self.name, [])
urls = urls + external_urls

for i, url in enumerate(urls):
# if url['product_id'] != '439806207' and i % 20 != 0:
# continue
url = 'https://mobileapi.n11.com/mobileapi/rest/getProductNonPersonalizedDetail?productId={}&vehicleType=&subChannel=N11&productDetailType=N11&adBidding=false'.format(
url['product_id'])
crawling_input_queue.add_url(CrawlingMeta(initial_data={'retailer': self.name,
'url': url,
'brand': 'from_evkiba_masterdata',
'category': 'from_evkiba_masterdata',
'subcategory': 'from_evkiba_masterdata',
'category_type': 'from_evkiba_masterdata',
'category_url': 'from_evkiba_masterdata',
},
url=url,
spider=copy.deepcopy(spider),
request_type='json',
request_method='get'))

def generate_sku_data(self, meta, queue, parse_field):
parse_field(meta, queue, 'product_availability')
if not meta.data.product_availability:
# print(meta.url.split('productId=')[1].split('&vehicleType')[0], 'GDATA::', meta.response.content)
print('OOS_PRODUCT::', meta.url.split('productId=')[1].split('&vehicleType')[0])
return

parse_field(meta, queue, 'options')
parse_field(meta, queue, 'product_url')
parse_field(meta, queue, 'product_image_url')
parse_field(meta, queue, 'merchant_name')
parse_field(meta, queue, 'brand')
parse_field(meta, queue, 'product_title')
parse_field(meta, queue, 'product_id')
parse_field(meta, queue, 'product_subtitle')
parse_field(meta, queue, 'group_id')
parse_field(meta, queue, 'product_barcode')

# self.log_fields(meta, ['product_id'])

# if len(meta.data.options) > 1:
# self.generate_option_data(meta, queue, parse_field)
# return

parse_field(meta, queue, 'product_barcode')
setattr(meta.data, 'product_price_2', None)
setattr(meta.data, 'promotion_text', None)
parse_field(meta, queue, 'product_availability')

if meta.data.product_availability:
setattr(meta.data, 'is_buybox', True)
setattr(meta.data, 'buybox_no', 1)
parse_field(meta, queue, 'product_price')
parse_field(meta, queue, 'cargo_fee')
parse_field(meta, queue, 'product_stock')

else:
setattr(meta.data, 'is_buybox', False)
setattr(meta.data, 'buybox_no', None)
setattr(meta.data, 'product_price', None)
setattr(meta.data, 'cargo_fee', None)
setattr(meta.data, 'product_stock', 0)
self.log_fields(meta, ['product_id', 'merchant_name', 'product_price', 'product_availability', 'is_buybox',
'buybox_no'])
self.add_oos_buybox(meta, meta.data, queue)

def generate_option_data(self, meta, queue, parse_field):

for option in meta.data.options:
parse_field(meta, queue, 'cargo_fee')
parse_field(meta, queue, 'product_availability')
parse_field(meta, queue, 'product_price')
option_meta = copy.deepcopy(meta)
option_meta.json = copy.deepcopy(option)

parse_field(option_meta, queue, 'option_id')
setattr(option_meta.data, 'product_id', option_meta.data.product_id + '__' + option_meta.data.option_id)

parse_field(option_meta, queue, 'option_title')
setattr(option_meta.data, 'product_title', option_meta.data.product_title + '__' + option_meta.data.option_title)

parse_field(option_meta, queue, 'option_barcode')
setattr(option_meta.data, 'product_barcode', option_meta.data.option_barcode)

setattr(option_meta.data, 'product_price_2', None)
setattr(option_meta.data, 'promotion_text', None)

# print('OPTION_DATA', option_meta.data)

if option_meta.data.product_availability:
setattr(option_meta.data, 'is_buybox', True)
setattr(option_meta.data, 'buybox_no', 1)
parse_field(option_meta, queue, 'option_price')
if option_meta.data.option_price is not None:
setattr(option_meta.data, 'product_price', option_meta.data.option_price)
parse_field(option_meta, queue, 'product_stock')

else:
setattr(option_meta.data, 'is_buybox', False)
setattr(option_meta.data, 'buybox_no', None)
setattr(option_meta.data, 'product_price', None)
setattr(option_meta.data, 'cargo_fee', None)
setattr(option_meta.data, 'product_stock', 0)

self.add_oos_buybox(option_meta, option_meta.data, queue)


class N11PriceUrlSpider(N11PriceSpider):
def __init__(self, delay):
super().__init__(delay)
self.delay = delay

self.callback_functions = [
self.generate_sku_data
]
self.product_id = {
'json': {
1: [{'key': 'product'}, {'key': 'id'}]
},
'process': lambda meta, unp_data, source, no: str(unp_data)
},
self.description_text = {
'json': {
1: [{'key': 'product'}, {'key': 'productAttributes'}]
},
'process': self.parse_description_text
},

self.rating = {
'json': {
1: [{'key': 'product'}, {'key': 'score'}]
},
'process': lambda meta, unp_data, source, no: float(unp_data)
},
self.review_count = {
'json': {
1: [{'key': 'product'}, {'key': 'reviewCount'}]
},
'process': lambda meta, unp_data, source, no: int(unp_data)
},
self.rating_count = {
'json': {
1: [{'key': 'product'}, {'key': 'reviewCount'}]
},
'process': self.parse_image_urls
},

def parse_description_text(self, meta, unp_data, source, no):
description = ''
for item in unp_data:
description += str(item['key'] + item['value'])
return description

@staticmethod
def parse_image_urls(meta, unp_data, source, no):
image_urls = []
for key in unp_data:
image_urls.append(key['path'])
return image_urls


def generate_input_metas(self, input, spider, crawling_input_queue, db):

urls = get_price_product_urls(db, self.name)

for i, url_input in enumerate(urls):
# if url_input['product_id'] != '522598164':
# continue
url = 'https://mobileapi.n11.com/mobileapi/rest/getProductNonPersonalizedDetail?productId={}&vehicleType=&subChannel=N11&productDetailType=N11&adBidding=false'.format(
url_input['product_id'])

crawling_input_queue.add_url(CrawlingMeta(initial_data={'retailer': self.name,
'url': url,
'brand': url_input['brand'],
'category': url_input['category'],
'subcategory': url_input['subcategory'],
'category_type': url_input['category_type'],
'category_url': 'FROM_PRODUCT_URL_CRAWLING',},
url=url,
spider=copy.deepcopy(spider),
request_type='json',
request_method='get'))

def generate_sku_data(self, meta, queue, parse_field):
parse_field(meta, queue, 'product_availability')
if not meta.data.product_availability:
# print(meta.url.split('productId=')[1].split('&vehicleType')[0], 'GDATA::', meta.response.content)
print('OOS_PRODUCT::', meta.url.split('productId=')[1].split('&vehicleType')[0])
return

parse_field(meta, queue, 'options')
parse_field(meta, queue, 'product_url')
parse_field(meta, queue, 'product_image_url')
parse_field(meta, queue, 'merchant_name')
parse_field(meta, queue, 'brand')
parse_field(meta, queue, 'product_title')
parse_field(meta, queue, 'product_id')
parse_field(meta, queue, 'product_subtitle')
parse_field(meta, queue, 'group_id')
parse_field(meta, queue, 'product_barcode')

# self.log_fields(meta, ['product_id'])

# if len(meta.data.options) > 1:
# self.generate_option_data(meta, queue, parse_field)
# return

parse_field(meta, queue, 'product_barcode')
setattr(meta.data, 'product_price_2', None)
setattr(meta.data, 'promotion_text', None)
parse_field(meta, queue, 'product_availability')

if meta.data.product_availability:
setattr(meta.data, 'is_buybox', True)
setattr(meta.data, 'buybox_no', 1)
parse_field(meta, queue, 'product_price')
parse_field(meta, queue, 'cargo_fee')
parse_field(meta, queue, 'product_stock')

else:
setattr(meta.data, 'is_buybox', False)
setattr(meta.data, 'buybox_no', None)
setattr(meta.data, 'product_price', None)
setattr(meta.data, 'cargo_fee', None)
setattr(meta.data, 'product_stock', 0)
self.log_fields(meta, ['product_id', 'product_title', 'merchant_name', 'product_price', 'product_availability', 'is_buybox',
'buybox_no'])
if meta.data.product_title:
self.add_sku_data(meta, meta.data, queue)

def generate_option_data(self, meta, queue, parse_field):

for option in meta.data.options:
parse_field(meta, queue, 'cargo_fee')
parse_field(meta, queue, 'product_availability')
parse_field(meta, queue, 'product_price')
option_meta = copy.deepcopy(meta)
option_meta.json = copy.deepcopy(option)

# print('----')
# print()
# print(option_meta.json)
# print()
# print()
# print('----')
parse_field(option_meta, queue, 'option_id')
setattr(option_meta.data, 'product_id', option_meta.data.product_id + '__' + option_meta.data.option_id)

parse_field(option_meta, queue, 'option_title')
setattr(option_meta.data, 'product_title', option_meta.data.product_title + '__' + option_meta.data.option_title)

parse_field(option_meta, queue, 'option_barcode')
setattr(option_meta.data, 'product_barcode', option_meta.data.option_barcode)

setattr(option_meta.data, 'product_price_2', None)
setattr(option_meta.data, 'promotion_text', None)

# print('OPTION_DATA', option_meta.data)

if option_meta.data.product_availability:
setattr(option_meta.data, 'is_buybox', True)
setattr(option_meta.data, 'buybox_no', 1)
parse_field(option_meta, queue, 'option_price')
if option_meta.data.option_price is not None:
setattr(option_meta.data, 'product_price', option_meta.data.option_price)
parse_field(option_meta, queue, 'product_stock')

else:
setattr(option_meta.data, 'is_buybox', False)
setattr(option_meta.data, 'buybox_no', None)
setattr(option_meta.data, 'product_price', None)
setattr(option_meta.data, 'cargo_fee', None)
setattr(option_meta.data, 'product_stock', 0)

self.add_sku_data(option_meta, option_meta.data, queue)

class N11ContentUrlSpider(N11PriceUrlSpider):
def __init__(self, delay):
super().__init__(delay)

self.callback_functions = [
self.generate_sku_data
]


def generate_input_metas(self, input, spider, crawling_input_queue, db):

crawling_input_queue.add_url(CrawlingMeta(initial_data=input,
url= 'https://mobileapi.n11.com/mobileapi/rest/getProductNonPersonalizedDetail?productId=508357220&vehicleType=&subChannel=N11&productDetailType=N11&adBidding=false',
spider=copy.deepcopy(spider),
request_type='json',
request_method='get'))
# def generate_input_metas(self, input, spider, crawling_input_queue, db):
#
# urls = get_content_product_urls(db, self.name)
#
#
# # for url in urls:
# # if url['product_id'] != '522598164':
# # continue
# # url = 'https://mobileapi.n11.com/mobileapi/rest/getProductNonPersonalizedDetail?productId={}&vehicleType=&subChannel=N11&productDetailType=N11&adBidding=false'.format(
# # url['product_id'])
# url = '508357220'
# crawling_input_queue.add_url(CrawlingMeta(initial_data={'retailer': self.name,
# 'url': 'https://mobileapi.n11.com/mobileapi/rest/getProductNonPersonalizedDetail?productId={}&vehicleType=&subChannel=N11&productDetailType=N11&adBidding=false'.format(
# url),
# 'brand': url['brand'],
# 'category': url['category'],
# 'subcategory': url['subcategory'],
# 'category_type': url['category_type'],
# 'category_url': 'FROM_PRODUCT_URL_CRAWLING',},
# url='https://mobileapi.n11.com/mobileapi/rest/getProductNonPersonalizedDetail?productId={}&vehicleType=&subChannel=N11&productDetailType=N11&adBidding=false'.format(
# url),
# spider=copy.deepcopy(spider),
# request_type='json',
# request_method='get'))


def generate_sku_data(self, meta, queue, parse_field):
parse_field(meta, queue, 'options')
parse_field(meta, queue, 'product_url')
parse_field(meta, queue, 'product_image_url')
parse_field(meta, queue, 'merchant_name')
parse_field(meta, queue, 'product_availability')
parse_field(meta, queue, 'brand')
parse_field(meta, queue, 'product_title')
parse_field(meta, queue, 'product_id')
parse_field(meta, queue, 'product_subtitle')
parse_field(meta, queue, 'group_id')
parse_field(meta, queue, 'product_barcode')
parse_field(meta, queue, 'review_count')
parse_field(meta, queue, 'rating')
parse_field(meta, queue, 'rating_count')
setattr(meta.data, 'image_urls', [meta.data.product_image_url] if meta.data.product_image_url else [])
setattr(meta.data, 'image_count', len(meta.data.image_urls))
setattr(meta.data, 'product_price_2', None)
setattr(meta.data, 'promotion_text', None)
parse_field(meta, queue, 'description_text')
setattr(meta.data, 'description_images', None)



if meta.data.brand in queue.brand_matching:
meta.data.brand = queue.brand_matching[meta.data.brand]

print('DATA:', meta.data.__dict__)
# print('DATA DETAILS:', meta.data_detail.__dict__)

if meta.data.product_availability:
print('ADDED:', meta.data.product_id, meta.data.product_url)
self.add_content_data(meta, queue)
else:
print('ELIMINATED_FROM_OOS:', meta.data.product_id, meta.data.product_url)