Index: pyyaml/trunk/lib/yaml/constructor.py =================================================================== --- pyyaml/trunk/lib/yaml/constructor.py (revision 198) +++ pyyaml/trunk/lib/yaml/constructor.py (revision 222) @@ -22,5 +22,5 @@ pass -class BaseConstructor: +class BaseConstructor(object): yaml_constructors = {} @@ -30,4 +30,6 @@ self.constructed_objects = {} self.recursive_objects = {} + self.state_generators = [] + self.deep_construct = False def check_data(self): @@ -40,53 +42,74 @@ return self.construct_document(self.get_node()) + def g(): yield None + generator_type = type(g()) + del g + def construct_document(self, node): data = self.construct_object(node) + while self.state_generators: + state_generators = self.state_generators + self.state_generators = [] + for generator in state_generators: + for dummy in generator: + pass self.constructed_objects = {} self.recursive_objects = {} + self.deep_construct = False return data - def construct_object(self, node): + def construct_object(self, node, deep=False): + if deep: + old_deep = self.deep_construct + self.deep_construct = True if node in self.constructed_objects: return self.constructed_objects[node] if node in self.recursive_objects: raise ConstructorError(None, None, - "found recursive node", node.start_mark) + "found unconstructable recursive node", node.start_mark) self.recursive_objects[node] = None constructor = None + state_constructor = None + tag_suffix = None if node.tag in self.yaml_constructors: - constructor = lambda node: self.yaml_constructors[node.tag](self, node) + constructor = self.yaml_constructors[node.tag] else: for tag_prefix in self.yaml_multi_constructors: if node.tag.startswith(tag_prefix): tag_suffix = node.tag[len(tag_prefix):] - constructor = lambda node: \ - self.yaml_multi_constructors[tag_prefix](self, tag_suffix, node) + constructor = self.yaml_multi_constructors[tag_prefix] break else: if None in self.yaml_multi_constructors: - constructor = lambda node: \ - self.yaml_multi_constructors[None](self, node.tag, node) + tag_suffix = node.tag + constructor = self.yaml_multi_constructors[None] elif None in self.yaml_constructors: - constructor = lambda node: \ - self.yaml_constructors[None](self, node) + constructor = self.yaml_constructors[None] elif isinstance(node, ScalarNode): - constructor = self.construct_scalar + constructor = self.__class__.construct_scalar elif isinstance(node, SequenceNode): - constructor = self.construct_sequence + constructor = self.__class__.construct_sequence elif isinstance(node, MappingNode): - constructor = self.construct_mapping - else: - print node.tag - data = constructor(node) + constructor = self.__class__.construct_mapping + if tag_suffix is None: + data = constructor(self, node) + else: + data = constructor(self, tag_suffix, node) + if isinstance(data, self.generator_type): + generator = data + data = generator.next() + if self.deep_construct: + for dummy in generator: + pass + else: + self.state_generators.append(generator) self.constructed_objects[node] = data del self.recursive_objects[node] + if deep: + self.deep_construct = old_deep return data def construct_scalar(self, node): if not isinstance(node, ScalarNode): - if isinstance(node, MappingNode): - for key_node in node.value: - if key_node.tag == u'tag:yaml.org,2002:value': - return self.construct_scalar(node.value[key_node]) raise ConstructorError(None, None, "expected a scalar node, but found %s" % node.id, @@ -94,12 +117,13 @@ return node.value - def construct_sequence(self, node): + def construct_sequence(self, node, deep=False): if not isinstance(node, SequenceNode): raise ConstructorError(None, None, "expected a sequence node, but found %s" % node.id, node.start_mark) - return [self.construct_object(child) for child in node.value] - - def construct_mapping(self, node): + return [self.construct_object(child, deep=deep) + for child in node.value] + + def construct_mapping(self, node, deep=False): if not isinstance(node, MappingNode): raise ConstructorError(None, None, @@ -107,15 +131,60 @@ node.start_mark) mapping = {} - merge = None - for key_node in node.value: + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError, exc: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unacceptable key (%s)" % exc, key_node.start_mark) + value = self.construct_object(value_node, deep=deep) + mapping[key] = value + return mapping + + def construct_pairs(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + pairs = [] + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + value = self.construct_object(value_node, deep=deep) + pairs.append((key, value)) + return pairs + + def add_constructor(cls, tag, constructor): + if not 'yaml_constructors' in cls.__dict__: + cls.yaml_constructors = cls.yaml_constructors.copy() + cls.yaml_constructors[tag] = constructor + add_constructor = classmethod(add_constructor) + + def add_multi_constructor(cls, tag_prefix, multi_constructor): + if not 'yaml_multi_constructors' in cls.__dict__: + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() + cls.yaml_multi_constructors[tag_prefix] = multi_constructor + add_multi_constructor = classmethod(add_multi_constructor) + +class SafeConstructor(BaseConstructor): + + def construct_scalar(self, node): + if isinstance(node, MappingNode): + for key_node, value_node in node.value: + if key_node.tag == u'tag:yaml.org,2002:value': + return self.construct_scalar(value_node) + return BaseConstructor.construct_scalar(self, node) + + def flatten_mapping(self, node): + merge = [] + index = 0 + while index < len(node.value): + key_node, value_node = node.value[index] if key_node.tag == u'tag:yaml.org,2002:merge': - if merge is not None: - raise ConstructorError("while constructing a mapping", node.start_mark, - "found duplicate merge key", key_node.start_mark) - value_node = node.value[key_node] + del node.value[index] if isinstance(value_node, MappingNode): - merge = [self.construct_mapping(value_node)] + self.flatten_mapping(value_node) + merge.extend(value_node.value) elif isinstance(value_node, SequenceNode): - merge = [] + submerge = [] for subnode in value_node.value: if not isinstance(subnode, MappingNode): @@ -124,6 +193,9 @@ "expected a mapping for merging, but found %s" % subnode.id, subnode.start_mark) - merge.append(self.construct_mapping(subnode)) - merge.reverse() + self.flatten_mapping(subnode) + submerge.append(subnode.value) + submerge.reverse() + for value in submerge: + merge.extend(value) else: raise ConstructorError("while constructing a mapping", node.start_mark, @@ -131,53 +203,15 @@ % value_node.id, value_node.start_mark) elif key_node.tag == u'tag:yaml.org,2002:value': - if '=' in mapping: - raise ConstructorError("while construction a mapping", node.start_mark, - "found duplicate value key", key_node.start_mark) - value = self.construct_object(node.value[key_node]) - mapping['='] = value + key_node.tag = u'tag:yaml.org,2002:str' + index += 1 else: - key = self.construct_object(key_node) - try: - duplicate_key = key in mapping - except TypeError, exc: - raise ConstructorError("while constructing a mapping", node.start_mark, - "found unacceptable key (%s)" % exc, key_node.start_mark) - if duplicate_key: - raise ConstructorError("while constructing a mapping", node.start_mark, - "found duplicate key", key_node.start_mark) - value = self.construct_object(node.value[key_node]) - mapping[key] = value - if merge is not None: - merge.append(mapping) - mapping = {} - for submapping in merge: - mapping.update(submapping) - return mapping - - def construct_pairs(self, node): - if not isinstance(node, MappingNode): - raise ConstructorError(None, None, - "expected a mapping node, but found %s" % node.id, - node.start_mark) - pairs = [] - for key_node in node.value: - key = self.construct_object(key_node) - value = self.construct_object(node.value[key_node]) - pairs.append((key, value)) - return pairs - - def add_constructor(cls, tag, constructor): - if not 'yaml_constructors' in cls.__dict__: - cls.yaml_constructors = cls.yaml_constructors.copy() - cls.yaml_constructors[tag] = constructor - add_constructor = classmethod(add_constructor) - - def add_multi_constructor(cls, tag_prefix, multi_constructor): - if not 'yaml_multi_constructors' in cls.__dict__: - cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() - cls.yaml_multi_constructors[tag_prefix] = multi_constructor - add_multi_constructor = classmethod(add_multi_constructor) - -class SafeConstructor(BaseConstructor): + index += 1 + if merge: + node.value = merge + node.value + + def construct_mapping(self, node, deep=False): + if isinstance(node, MappingNode): + self.flatten_mapping(node) + return BaseConstructor.construct_mapping(self, node, deep=deep) def construct_yaml_null(self, node): @@ -297,8 +331,9 @@ # Note: we do not check for duplicate keys, because it's too # CPU-expensive. + omap = [] + yield omap if not isinstance(node, SequenceNode): raise ConstructorError("while constructing an ordered map", node.start_mark, "expected a sequence, but found %s" % node.id, node.start_mark) - omap = [] for subnode in node.value: if not isinstance(subnode, MappingNode): @@ -310,16 +345,16 @@ "expected a single mapping item, but found %d items" % len(subnode.value), subnode.start_mark) - key_node = subnode.value.keys()[0] + key_node, value_node = subnode.value[0] key = self.construct_object(key_node) - value = self.construct_object(subnode.value[key_node]) + value = self.construct_object(value_node) omap.append((key, value)) - return omap def construct_yaml_pairs(self, node): # Note: the same code as `construct_yaml_omap`. + pairs = [] + yield pairs if not isinstance(node, SequenceNode): raise ConstructorError("while constructing pairs", node.start_mark, "expected a sequence, but found %s" % node.id, node.start_mark) - pairs = [] for subnode in node.value: if not isinstance(subnode, MappingNode): @@ -331,13 +366,14 @@ "expected a single mapping item, but found %d items" % len(subnode.value), subnode.start_mark) - key_node = subnode.value.keys()[0] + key_node, value_node = subnode.value[0] key = self.construct_object(key_node) - value = self.construct_object(subnode.value[key_node]) + value = self.construct_object(value_node) pairs.append((key, value)) - return pairs def construct_yaml_set(self, node): + data = set() + yield data value = self.construct_mapping(node) - return set(value) + data.update(value) def construct_yaml_str(self, node): @@ -349,17 +385,23 @@ def construct_yaml_seq(self, node): - return self.construct_sequence(node) + data = [] + yield data + data.extend(self.construct_sequence(node)) def construct_yaml_map(self, node): - return self.construct_mapping(node) + data = {} + yield data + value = self.construct_mapping(node) + data.update(value) def construct_yaml_object(self, node, cls): - state = self.construct_mapping(node) data = cls.__new__(cls) + yield data if hasattr(data, '__setstate__'): + state = self.construct_mapping(node, deep=True) data.__setstate__(state) else: + state = self.construct_mapping(node) data.__dict__.update(state) - return data def construct_undefined(self, node): @@ -435,5 +477,5 @@ def construct_python_tuple(self, node): - return tuple(self.construct_yaml_seq(node)) + return tuple(self.construct_sequence(node)) def find_python_module(self, name, mark): @@ -526,7 +568,8 @@ # !!python/object:module.name { ... state ... } instance = self.make_python_instance(suffix, node, newobj=True) - state = self.construct_mapping(node) + yield instance + deep = hasattr(instance, '__setstate__') + state = self.construct_mapping(node, deep=deep) self.set_python_instance_state(instance, state) - return instance def construct_python_object_apply(self, suffix, node, newobj=False): @@ -543,5 +586,5 @@ # is how an object is created, check make_python_instance for details. if isinstance(node, SequenceNode): - args = self.construct_sequence(node) + args = self.construct_sequence(node, deep=True) kwds = {} state = {} @@ -549,5 +592,5 @@ dictitems = {} else: - value = self.construct_mapping(node) + value = self.construct_mapping(node, deep=True) args = value.get('args', []) kwds = value.get('kwds', {}) @@ -568,5 +611,4 @@ return self.construct_python_object_apply(suffix, node, newobj=True) - Constructor.add_constructor( u'tag:yaml.org,2002:python/none',