我构建了一个网络抓取工具this https://campus.datacamp.com/courses/intro-to-python-for-data-science/chapter-1-python-basics该页面取决于将字符串解析为 JSON 文件。但他们对该网站进行了一些更新，现在抓取工具已经停止工作。我认为问题在于我需要的信息不再是 JSON 结构。
这是我原来的：
# Packages
from bs4 import BeautifulSoup
from urllib.request import urlopen, urlretrieve
import json
import ast
# The part that still works
address = 'https://campus.datacamp.com/courses/intro-to-python-for-data-science/chapter-1-python-basics?ex=2'
html = urlopen(address)
soup = BeautifulSoup(html, 'lxml')
string = soup.find_all('script')[2].string
json_text = string.strip('window.PRELOADED_STATE = "')[:-2]
# The part that's now broken
lesson = json.loads(json_text)
#> Traceback (most recent call last):
#> <ipython-input-11-f9b7d249d994> in <module>()
#> 2 # The part that's now broken
#> 3
#> ----> 4 lesson = json.loads(json_text)
#> ~/anaconda3/lib/python3.6/json/__init__.py in loads(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
#> 352 parse_int is None and parse_float is None and
#> 353 parse_constant is None and object_pairs_hook is None and not kw):
#> --> 354 return _default_decoder.decode(s)
#> 355 if cls is None:
#> 356 cls = JSONDecoder
#> ~/anaconda3/lib/python3.6/json/decoder.py in decode(self, s, _w)
#> 337
#> 338 """
#> --> 339 obj, end = self.raw_decode(s, idx=_w(s, 0).end())
#> 340 end = _w(s, end).end()
#> 341 if end != len(s):
#> ~/anaconda3/lib/python3.6/json/decoder.py in raw_decode(self, s, idx)
#> 355 obj, end = self.scan_once(s, idx)
#> 356 except StopIteration as err:
#> --> 357 raise JSONDecodeError("Expecting value", s, err.value) from None
#> 358 return obj, end
#> JSONDecodeError: Expecting value: line 1 column 2 (char 1)
问题是所有的信息都在json_text不再采用 JSON 结构。
need_to_parse = BeautifulSoup(json_text, 'lxml').string #Escape HTML
print(len(need_to_parse))
#> 61453
print(need_to_parse[:50])
#> ["~#iM",["preFetchedData",["^0",["course",["^0",["
print(need_to_parse[-50:])
#> "type","MultipleChoiceExercise","id",14253]]]]]]]]
我想也许是一个嵌套列表，所以我尝试了ast.literal_eval()，但没有运气！
parsed_list = ast.literal_eval(need_to_parse)
#> Traceback (most recent call last):
#> File "/Users/nicholascifuentes-goodbody/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2862, in run_code
#> exec(code_obj, self.user_global_ns, self.user_ns)
#> File "<ipython-input-13-55b60da762d6>", line 2, in <module>
#> parsed_list = ast.literal_eval(need_to_parse)
#> File "/Users/nicholascifuentes-goodbody/anaconda3/lib/python3.6/ast.py", line 48, in literal_eval
#> node_or_string = parse(node_or_string, mode='eval')
#> File "/Users/nicholascifuentes-goodbody/anaconda3/lib/python3.6/ast.py", line 35, in parse
#> return compile(source, filename, mode, PyCF_ONLY_AST)
#> File "<unknown>", line 1
#> ["~#iM",["preFetchedData"
完整的输出位于txt file HERE https://www.dropbox.com/s/jwv03bekok9tnr2/output.txt?dl=0.
有人认识这个数据结构吗？解析它的最佳方法是什么？
创建于 2018-10-19reprexpy 包 https://github.com/crew102/reprexpy
import reprexpy
print(reprexpy.SessionInfo())
#> Session info --------------------------------------------------------------------
#> Platform: Darwin-17.7.0-x86_64-i386-64bit (64-bit)
#> Python: 3.6
#> Date: 2018-10-19
#> Packages ------------------------------------------------------------------------
#> beautifulsoup4==4.6.0
#> reprexpy==0.1.1
