1__all__ = [
2    'HTML5TreeBuilder',
3    ]
4
5import warnings
6from bs4.builder import (
7    PERMISSIVE,
8    HTML,
9    HTML_5,
10    HTMLTreeBuilder,
11    )
12from bs4.element import (
13    NamespacedAttribute,
14    whitespace_re,
15)
16import html5lib
17try:
18    # html5lib >= 0.99999999/1.0b9
19    from html5lib.treebuilders import base as treebuildersbase
20except ImportError:
21    # html5lib <= 0.9999999/1.0b8
22    from html5lib.treebuilders import _base as treebuildersbase
23from html5lib.constants import namespaces
24
25from bs4.element import (
26    Comment,
27    Doctype,
28    NavigableString,
29    Tag,
30    )
31
32class HTML5TreeBuilder(HTMLTreeBuilder):
33    """Use html5lib to build a tree."""
34
35    NAME = "html5lib"
36
37    features = [NAME, PERMISSIVE, HTML_5, HTML]
38
39    def prepare_markup(self, markup, user_specified_encoding,
40                       document_declared_encoding=None, exclude_encodings=None):
41        # Store the user-specified encoding for use later on.
42        self.user_specified_encoding = user_specified_encoding
43
44        # document_declared_encoding and exclude_encodings aren't used
45        # ATM because the html5lib TreeBuilder doesn't use
46        # UnicodeDammit.
47        if exclude_encodings:
48            warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
49        yield (markup, None, None, False)
50
51    # These methods are defined by Beautiful Soup.
52    def feed(self, markup):
53        if self.soup.parse_only is not None:
54            warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
55        parser = html5lib.HTMLParser(tree=self.create_treebuilder)
56        doc = parser.parse(markup, encoding=self.user_specified_encoding)
57
58        # Set the character encoding detected by the tokenizer.
59        if isinstance(markup, str):
60            # We need to special-case this because html5lib sets
61            # charEncoding to UTF-8 if it gets Unicode input.
62            doc.original_encoding = None
63        else:
64            doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
65
66    def create_treebuilder(self, namespaceHTMLElements):
67        self.underlying_builder = TreeBuilderForHtml5lib(
68            self.soup, namespaceHTMLElements)
69        return self.underlying_builder
70
71    def test_fragment_to_document(self, fragment):
72        """See `TreeBuilder`."""
73        return '<html><head></head><body>%s</body></html>' % fragment
74
75
76class TreeBuilderForHtml5lib(treebuildersbase.TreeBuilder):
77
78    def __init__(self, soup, namespaceHTMLElements):
79        self.soup = soup
80        super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
81
82    def documentClass(self):
83        self.soup.reset()
84        return Element(self.soup, self.soup, None)
85
86    def insertDoctype(self, token):
87        name = token["name"]
88        publicId = token["publicId"]
89        systemId = token["systemId"]
90
91        doctype = Doctype.for_name_and_ids(name, publicId, systemId)
92        self.soup.object_was_parsed(doctype)
93
94    def elementClass(self, name, namespace):
95        tag = self.soup.new_tag(name, namespace)
96        return Element(tag, self.soup, namespace)
97
98    def commentClass(self, data):
99        return TextNode(Comment(data), self.soup)
100
101    def fragmentClass(self):
102        self.soup = BeautifulSoup("")
103        self.soup.name = "[document_fragment]"
104        return Element(self.soup, self.soup, None)
105
106    def appendChild(self, node):
107        # XXX This code is not covered by the BS4 tests.
108        self.soup.append(node.element)
109
110    def getDocument(self):
111        return self.soup
112
113    def getFragment(self):
114        return treebuildersbase.TreeBuilder.getFragment(self).element
115
116class AttrList(object):
117    def __init__(self, element):
118        self.element = element
119        self.attrs = dict(self.element.attrs)
120    def __iter__(self):
121        return list(self.attrs.items()).__iter__()
122    def __setitem__(self, name, value):
123        # If this attribute is a multi-valued attribute for this element,
124        # turn its value into a list.
125        list_attr = HTML5TreeBuilder.cdata_list_attributes
126        if (name in list_attr['*']
127            or (self.element.name in list_attr
128                and name in list_attr[self.element.name])):
129            # A node that is being cloned may have already undergone
130            # this procedure.
131            if not isinstance(value, list):
132                value = whitespace_re.split(value)
133        self.element[name] = value
134    def items(self):
135        return list(self.attrs.items())
136    def keys(self):
137        return list(self.attrs.keys())
138    def __len__(self):
139        return len(self.attrs)
140    def __getitem__(self, name):
141        return self.attrs[name]
142    def __contains__(self, name):
143        return name in list(self.attrs.keys())
144
145
146class Element(treebuildersbase.Node):
147    def __init__(self, element, soup, namespace):
148        treebuildersbase.Node.__init__(self, element.name)
149        self.element = element
150        self.soup = soup
151        self.namespace = namespace
152
153    def appendChild(self, node):
154        string_child = child = None
155        if isinstance(node, str):
156            # Some other piece of code decided to pass in a string
157            # instead of creating a TextElement object to contain the
158            # string.
159            string_child = child = node
160        elif isinstance(node, Tag):
161            # Some other piece of code decided to pass in a Tag
162            # instead of creating an Element object to contain the
163            # Tag.
164            child = node
165        elif node.element.__class__ == NavigableString:
166            string_child = child = node.element
167        else:
168            child = node.element
169
170        if not isinstance(child, str) and child.parent is not None:
171            node.element.extract()
172
173        if (string_child and self.element.contents
174            and self.element.contents[-1].__class__ == NavigableString):
175            # We are appending a string onto another string.
176            # TODO This has O(n^2) performance, for input like
177            # "a</a>a</a>a</a>..."
178            old_element = self.element.contents[-1]
179            new_element = self.soup.new_string(old_element + string_child)
180            old_element.replace_with(new_element)
181            self.soup._most_recent_element = new_element
182        else:
183            if isinstance(node, str):
184                # Create a brand new NavigableString from this string.
185                child = self.soup.new_string(node)
186
187            # Tell Beautiful Soup to act as if it parsed this element
188            # immediately after the parent's last descendant. (Or
189            # immediately after the parent, if it has no children.)
190            if self.element.contents:
191                most_recent_element = self.element._last_descendant(False)
192            elif self.element.next_element is not None:
193                # Something from further ahead in the parse tree is
194                # being inserted into this earlier element. This is
195                # very annoying because it means an expensive search
196                # for the last element in the tree.
197                most_recent_element = self.soup._last_descendant()
198            else:
199                most_recent_element = self.element
200
201            self.soup.object_was_parsed(
202                child, parent=self.element,
203                most_recent_element=most_recent_element)
204
205    def getAttributes(self):
206        return AttrList(self.element)
207
208    def setAttributes(self, attributes):
209
210        if attributes is not None and len(attributes) > 0:
211
212            converted_attributes = []
213            for name, value in list(attributes.items()):
214                if isinstance(name, tuple):
215                    new_name = NamespacedAttribute(*name)
216                    del attributes[name]
217                    attributes[new_name] = value
218
219            self.soup.builder._replace_cdata_list_attribute_values(
220                self.name, attributes)
221            for name, value in list(attributes.items()):
222                self.element[name] = value
223
224            # The attributes may contain variables that need substitution.
225            # Call set_up_substitutions manually.
226            #
227            # The Tag constructor called this method when the Tag was created,
228            # but we just set/changed the attributes, so call it again.
229            self.soup.builder.set_up_substitutions(self.element)
230    attributes = property(getAttributes, setAttributes)
231
232    def insertText(self, data, insertBefore=None):
233        if insertBefore:
234            text = TextNode(self.soup.new_string(data), self.soup)
235            self.insertBefore(data, insertBefore)
236        else:
237            self.appendChild(data)
238
239    def insertBefore(self, node, refNode):
240        index = self.element.index(refNode.element)
241        if (node.element.__class__ == NavigableString and self.element.contents
242            and self.element.contents[index-1].__class__ == NavigableString):
243            # (See comments in appendChild)
244            old_node = self.element.contents[index-1]
245            new_str = self.soup.new_string(old_node + node.element)
246            old_node.replace_with(new_str)
247        else:
248            self.element.insert(index, node.element)
249            node.parent = self
250
251    def removeChild(self, node):
252        node.element.extract()
253
254    def reparentChildren(self, new_parent):
255        """Move all of this tag's children into another tag."""
256        # print "MOVE", self.element.contents
257        # print "FROM", self.element
258        # print "TO", new_parent.element
259        element = self.element
260        new_parent_element = new_parent.element
261        # Determine what this tag's next_element will be once all the children
262        # are removed.
263        final_next_element = element.next_sibling
264
265        new_parents_last_descendant = new_parent_element._last_descendant(False, False)
266        if len(new_parent_element.contents) > 0:
267            # The new parent already contains children. We will be
268            # appending this tag's children to the end.
269            new_parents_last_child = new_parent_element.contents[-1]
270            new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
271        else:
272            # The new parent contains no children.
273            new_parents_last_child = None
274            new_parents_last_descendant_next_element = new_parent_element.next_element
275
276        to_append = element.contents
277        append_after = new_parent_element.contents
278        if len(to_append) > 0:
279            # Set the first child's previous_element and previous_sibling
280            # to elements within the new parent
281            first_child = to_append[0]
282            if new_parents_last_descendant:
283                first_child.previous_element = new_parents_last_descendant
284            else:
285                first_child.previous_element = new_parent_element
286            first_child.previous_sibling = new_parents_last_child
287            if new_parents_last_descendant:
288                new_parents_last_descendant.next_element = first_child
289            else:
290                new_parent_element.next_element = first_child
291            if new_parents_last_child:
292                new_parents_last_child.next_sibling = first_child
293
294            # Fix the last child's next_element and next_sibling
295            last_child = to_append[-1]
296            last_child.next_element = new_parents_last_descendant_next_element
297            if new_parents_last_descendant_next_element:
298                new_parents_last_descendant_next_element.previous_element = last_child
299            last_child.next_sibling = None
300
301        for child in to_append:
302            child.parent = new_parent_element
303            new_parent_element.contents.append(child)
304
305        # Now that this element has no children, change its .next_element.
306        element.contents = []
307        element.next_element = final_next_element
308
309        # print "DONE WITH MOVE"
310        # print "FROM", self.element
311        # print "TO", new_parent_element
312
313    def cloneNode(self):
314        tag = self.soup.new_tag(self.element.name, self.namespace)
315        node = Element(tag, self.soup, self.namespace)
316        for key,value in self.attributes:
317            node.attributes[key] = value
318        return node
319
320    def hasContent(self):
321        return self.element.contents
322
323    def getNameTuple(self):
324        if self.namespace is None:
325            return namespaces["html"], self.name
326        else:
327            return self.namespace, self.name
328
329    nameTuple = property(getNameTuple)
330
331class TextNode(Element):
332    def __init__(self, element, soup):
333        treebuildersbase.Node.__init__(self, None)
334        self.element = element
335        self.soup = soup
336
337    def cloneNode(self):
338        raise NotImplementedError
339