selenium - Python Scraping - Unable to get required data from Flipkart -
i trying scrape customer reviews flipkart website. following link. following code scrape, returning empty list.
>>> bs4 import beautifulsoup >>> import requests >>> r = requests.get('https://www.flipkart.com/samsung-galaxy-j5-6-new-2016-edition-white-16-gb/product-reviews/itmegmrnzqjcpfg9?pid=mobeg4xwjg7f9a6z') >>> soup = beautifulsoup(r.content, 'lxml') # tried 'html.parser' >>> soup.find_all('div', '_3dcdkt') [] >>> soup.find_all('div', {'class': '_3dcdkt'}) [] >>> soup.find_all('div', {'class': 'row _3wyu6i _3brc7l'}) [] >>> soup.find_all('div', {'class': '_1grhlx hfpo14'}) [] so, tried entire section, getting following:
>>> soup.find_all('div', {'class': 'col-9-12'}) [<div class="col-9-12" data-reactid="96"><div class="row _2_xtr5" data-reactid="97"></div><div class="row _3wyu6i _1kvtzt" data-reactid="98"></div></div>] i not getting other contents. so, next tried selenium, returning none. following selenium code:
>>> driver = webdriver.firefox() >>> driver.get('https://www.flipkart.com/samsung-galaxy-j5-6-new-2016-edition-white-16-gb/product-reviews/itmegmrnzqjcpfg9?pid=mobeg4xwjg7f9a6z') >>> = driver.find_elements_by_class_name("_3dcdkt") >>> len(a) 10 >>> in a: ... print i.get_attribute('value') ... none none none none none none none none none none what might problem? doing mistakes in code. kindly help. new python.
the reviews etc.. populated using reactjs, data retrieved using ajax request can mimic requests:
import requests data = {"productid": "mobeg4xwjg7f9a6z", # end of url pid=mobeg4xwjg7f9a6z "count": "15", "ratings": "all", "reviewertype:all" "sortorder": "most_helpful"} headers = ({"x-user-agent":"mozilla/5.0 (x11; linux x86_64) applewebkit/537.36 (khtml, gecko) chrome/53.0.2785.92 safari/537.36 fkua/website/41/website/desktop"}) data = requests.get("https://www.flipkart.com/api/3/product/reviews", params=data, headers=headers).json() print(data) what want access data["response"]["data"] list of dicts:
for dct in data["response"]["data"] print(dct) which give you:
{u'action': none, u'fixed': false, u'value': {u'rating': 5, u'text': u'thanks flipkart deliver me in 5 days \ngood phone metal body \nand best front camera flash\nbest night selfie \ni take more 30 pic in night mode front flash \ngood smartphone gold color supereb\nbest ever smartphone under 15k samsung\ngood battery\ngood camera front flash , rear superb', u'reportabuse': {u'action': {u'originalurl': none, u'params': {u'vote': u'abuse', u'reviewid': u'be37810e-20fe-4417-9d88-2709288cf2ba', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'totalcount': 285, u'downvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'down', u'reviewid': u'be37810e-20fe-4417-9d88-2709288cf2ba', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 74, u'type': u'votevalue'}, u'tracking': none}, u'id': u'be37810e-20fe-4417-9d88-2709288cf2ba', u'author': u'happy thakur', u'url': u'/reviews/be37810e-20fe-4417-9d88-2709288cf2ba', u'upvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'up', u'reviewid': u'be37810e-20fe-4417-9d88-2709288cf2ba', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 211, u'type': u'votevalue'}, u'tracking': none}, u'helpfulcount': 211, u'created': u'16 may, 2016', u'certifiedbuyer': true, u'title': u'best smartphone samsung', u'type': u'productreviewvalue'}, u'tracking': none} {u'action': none, u'fixed': false, u'value': {u'rating': 5, u'text': u"updated review on 02-august after 3 months of usage:\nwhat liked most:\nlook : 100/100 - looking phone. gold color , finishing super cool\nsize : 100/100 - 5.2 inch neither big nor small. can still operate 1 hand.. \nbattery : 100/100 - 3100 mah outstanding. 3g on when out of home , wi-fi on in home. charging mobile once in every 36 hours. use whatsapp, instagram , browsing mostly. \ndisplay : 90/100 - not bright , sharp s series phones, real deal price. impressed again. worry not having gorilla scratch proof glass. may need use tempered glass.\ntouch : 95/100 - smooth , dont see lags of now.\ncamera : 90/100 - photos , can capture fast, again not great s series phones. @ price believe phone outclasses other competitors in camera department. \n\none last thing samsung brand , service center coverage, again awesome. \noverall satisfied phone , phone reached expectations. \nwhat disliked:\nearphone jack @ bottom.. feel uncomfortable when chatting , listening songs @ same time\nlow speaker volume, not big deal though me, don't use loudspeaker songs mostly", u'reportabuse': {u'action': {u'originalurl': none, u'params': {u'vote': u'abuse', u'reviewid': u'e786669a-024b-4ef0-b70c-1e4fcf5fe5ff', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'totalcount': 272, u'downvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'down', u'reviewid': u'e786669a-024b-4ef0-b70c-1e4fcf5fe5ff', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 87, u'type': u'votevalue'}, u'tracking': none}, u'id': u'e786669a-024b-4ef0-b70c-1e4fcf5fe5ff', u'author': u'naresh kareti', u'url': u'/reviews/e786669a-024b-4ef0-b70c-1e4fcf5fe5ff', u'upvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'up', u'reviewid': u'e786669a-024b-4ef0-b70c-1e4fcf5fe5ff', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 185, u'type': u'votevalue'}, u'tracking': none}, u'helpfulcount': 185, u'created': u'13 may, 2016', u'certifiedbuyer': true, u'title': u'absolute stunner , impressive', u'type': u'productreviewvalue'}, u'tracking': none} {u'action': none, u'fixed': false, u'value': {u'rating': 3, u'text': u'hi,\n\ni got phone flipkart on friday , here 3 days review.\n\npros:\n * beautiful design\n * handy, easy handle\n * battery backup great\n * camera good\n * no heating issues\n \ncons:\n * if charging, not show light or notification whether charging or not. need on screen , check whether charging or not. every time need turn on , see whether charging or not.\n* camera issue: once take picture , press button taking time come camera mode.\n* if turn on flash , take pic camera taking time capture picture. out flash taking fast.\n* volume low. not enough medium sized room.\n* ear phones not me. \n\n\nwill post feedback after using 15 days.\n\nthanks', u'reportabuse': {u'action': {u'originalurl': none, u'params': {u'vote': u'abuse', u'reviewid': u'9cbcd27c-a8ad-4793-978a-5903cd086252', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'totalcount': 212, u'downvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'down', u'reviewid': u'9cbcd27c-a8ad-4793-978a-5903cd086252', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 67, u'type': u'votevalue'}, u'tracking': none}, u'id': u'9cbcd27c-a8ad-4793-978a-5903cd086252', u'author': u'ileep ', u'url': u'/reviews/9cbcd27c-a8ad-4793-978a-5903cd086252', u'upvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'up', u'reviewid': u'9cbcd27c-a8ad-4793-978a-5903cd086252', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 145, u'type': u'votevalue'}, u'tracking': none}, u'helpfulcount': 145, u'created': u'16 may, 2016', u'certifiedbuyer': true, u'title': u'good looking phone drawbacks', u'type': u'productreviewvalue'}, u'tracking': none} {u'action': none, u'fixed': false, u'value': {u'rating': 5, u'text': u'super amoled display..2 gb ram latest android marshmallow os 13k....its difficult samsung phone 2 gb ram in such low price range...used 15 days....going smooth....awesome earphone quality.....selfie , camera good.....battery last more day continous usage or go 2 days....free microsoft apps , more...', u'reportabuse': {u'action': {u'originalurl': none, u'params': {u'vote': u'abuse', u'reviewid': u'1546ed16-5945-4257-9f2d-0d86db7ed92e', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'totalcount': 34, u'downvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'down', u'reviewid': u'1546ed16-5945-4257-9f2d-0d86db7ed92e', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 9, u'type': u'votevalue'}, u'tracking': none}, u'id': u'1546ed16-5945-4257-9f2d-0d86db7ed92e', u'author': u'prashant dias', u'url': u'/reviews/1546ed16-5945-4257-9f2d-0d86db7ed92e', u'upvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'up', u'reviewid': u'1546ed16-5945-4257-9f2d-0d86db7ed92e', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 25, u'type': u'votevalue'}, u'tracking': none}, u'helpfulcount': 25, u'created': u'7 sep, 2016', u'certifiedbuyer': true, u'title': u'brilliant phone compared money', u'type': u'productreviewvalue'}, u'tracking': none} {u'action': none, u'fixed': false, u'value': {u'rating': 5, u'text': u"nice.battery backup it's good", u'reportabuse': {u'action': {u'originalurl': none, u'params': {u'vote': u'abuse', u'reviewid': u'a9f2f6a0-2272-4187-bd37-48eb8a0a85c9', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'totalcount': 5, u'downvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'down', u'reviewid': u'a9f2f6a0-2272-4187-bd37-48eb8a0a85c9', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'id': u'a9f2f6a0-2272-4187-bd37-48eb8a0a85c9', u'author': u'flipkart customer', u'url': u'/reviews/a9f2f6a0-2272-4187-bd37-48eb8a0a85c9', u'upvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'up', u'reviewid': u'a9f2f6a0-2272-4187-bd37-48eb8a0a85c9', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 5, u'type': u'votevalue'}, u'tracking': none}, u'helpfulcount': 5, u'created': u'17 aug, 2016', u'certifiedbuyer': true, u'title': u"it's good", u'type': u'productreviewvalue'}, u'tracking': none} {u'action': none, u'fixed': false, u'value': {u'rating': 5, u'text': u'this phone awesome..must buy', u'reportabuse': {u'action': {u'originalurl': none, u'params': {u'vote': u'abuse', u'reviewid': u'cf8cf2c8-1f79-4d56-a4cd-e641ffb3551b', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'totalcount': 5, u'downvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'down', u'reviewid': u'cf8cf2c8-1f79-4d56-a4cd-e641ffb3551b', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'id': u'cf8cf2c8-1f79-4d56-a4cd-e641ffb3551b', u'author': u'durvank aregekar', u'url': u'/reviews/cf8cf2c8-1f79-4d56-a4cd-e641ffb3551b', u'upvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'up', u'reviewid': u'cf8cf2c8-1f79-4d56-a4cd-e641ffb3551b', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 5, u'type': u'votevalue'}, u'tracking': none}, u'helpfulcount': 5, u'created': u'10 aug, 2016', u'certifiedbuyer': true, u'title': u'must buy', u'type': u'productreviewvalue'}, u'tracking': none} {u'action': none, u'fixed': false, u'value': {u'rating': 5, u'text': u'it phone', u'reportabuse': {u'action': {u'originalurl': none, u'params': {u'vote': u'abuse', u'reviewid': u'ce31beb5-5c8f-4a2d-be7d-aba416592df2', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'totalcount': 5, u'downvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'down', u'reviewid': u'ce31beb5-5c8f-4a2d-be7d-aba416592df2', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'id': u'ce31beb5-5c8f-4a2d-be7d-aba416592df2', u'author': u'sourabh jain', u'url': u'/reviews/ce31beb5-5c8f-4a2d-be7d-aba416592df2', u'upvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'up', u'reviewid': u'ce31beb5-5c8f-4a2d-be7d-aba416592df2', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 5, u'type': u'votevalue'}, u'tracking': none}, u'helpfulcount': 5, u'created': u'9 aug, 2016', u'certifiedbuyer': true, u'title': u'good phone', u'type': u'productreviewvalue'}, u'tracking': none} {u'action': none, u'fixed': false, u'value': {u'rating': 5, u'text': u'delivery in time phone heat data on plz check', u'reportabuse': {u'action': {u'originalurl': none, u'params': {u'vote': u'abuse', u'reviewid': u'1fcf5a13-edef-4b16-8372-8732819c143c', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'totalcount': 9, u'downvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'down', u'reviewid': u'1fcf5a13-edef-4b16-8372-8732819c143c', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 1, u'type': u'votevalue'}, u'tracking': none}, u'id': u'1fcf5a13-edef-4b16-8372-8732819c143c', u'author': u'santhoaha m n santhu', u'url': u'/reviews/1fcf5a13-edef-4b16-8372-8732819c143c', u'upvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'up', u'reviewid': u'1fcf5a13-edef-4b16-8372-8732819c143c', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 8, u'type': u'votevalue'}, u'tracking': none}, u'helpfulcount': 8, u'created': u'12 aug, 2016', u'certifiedbuyer': true, u'title': u'very good', u'type': u'productreviewvalue'}, u'tracking': none} {u'action': none, u'fixed': false, u'value': {u'rating': 4, u'text': u'good product samsung\n\nthe things phone is\n 1. marshmellow v6.0\n 2. front flash 5mb camera not good\n 3. design\n 4. primary camera not 13mb led flash\n 5. battery life not good\n 6. size correct in design\n 7. supports otg\n 9. 2gb ram\n 10. 16gb internal storage 11gb availiable\n 11. 4g supports\n 12. ultra power saving mode\n 13. s bike mode\n 14. speaker volume not good\n 15. 3g supports\n 16. ultra data saving\n 17. no auto brightness\n 18. 2g supports\n 19. top performance \n 20. phone @ price 14000\n *********************', u'reportabuse': {u'action': {u'originalurl': none, u'params': {u'vote': u'abuse', u'reviewid': u'958efa75-1b67-4872-9f71-b18035fafe6a', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'totalcount': 20, u'downvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'down', u'reviewid': u'958efa75-1b67-4872-9f71-b18035fafe6a', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 5, u'type': u'votevalue'}, u'tracking': none}, u'id': u'958efa75-1b67-4872-9f71-b18035fafe6a', u'author': u'vaishnav ', u'url': u'/reviews/958efa75-1b67-4872-9f71-b18035fafe6a', u'upvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'up', u'reviewid': u'958efa75-1b67-4872-9f71-b18035fafe6a', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 15, u'type': u'votevalue'}, u'tracking': none}, u'helpfulcount': 15, u'created': u'17 may, 2016', u'certifiedbuyer': true, u'title': u'by expert -vaishnav vj', u'type': u'productreviewvalue'}, u'tracking': none} {u'action': none, u'fixed': false, u'value': {u'rating': 4, u'text': u'very nice device', u'reportabuse': {u'action': {u'originalurl': none, u'params': {u'vote': u'abuse', u'reviewid': u'c7177dfb-39c2-4c0b-8bbd-288f96757c3a', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'totalcount': 4, u'downvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'down', u'reviewid': u'c7177dfb-39c2-4c0b-8bbd-288f96757c3a', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 0, u'type': u'votevalue'}, u'tracking': none}, u'id': u'c7177dfb-39c2-4c0b-8bbd-288f96757c3a', u'author': u'flipkart customer', u'url': u'/reviews/c7177dfb-39c2-4c0b-8bbd-288f96757c3a', u'upvote': {u'action': {u'originalurl': none, u'params': {u'vote': u'up', u'reviewid': u'c7177dfb-39c2-4c0b-8bbd-288f96757c3a', u'reviewdomain': u'product'}, u'logintype': u'legacy_login', u'url': none, u'fallback': none, u'type': u'review_vote', u'omnituredata': none, u'screentype': none, u'tracking': {}}, u'fixed': false, u'value': {u'count': 4, u'type': u'votevalue'}, u'tracking': none}, u'helpfulcount': 4, u'created': u'8 sep, 2016', u'certifiedbuyer': true, u'title': u'good quality product', u'type': u'productreviewvalue'}, u'tracking': none} the x-user-agent required, without 403. can play around parameters see different results, leave you.
Comments
Post a Comment