Python: Replace values in nested dictionary

The following piece of code replaces (substrings of) values in a dictionary. It works for nested json structures and copes with json, list and string types. You can easily add other types if needed.

def dict_replace_value(d: dict, old: str, new: str) -> dict:
    x = {}
    for k, v in d.items():
        if isinstance(v, dict):
            v = dict_replace_value(v, old, new)
        elif isinstance(v, list):
            v = list_replace_value(v, old, new)
        elif isinstance(v, str):
            v = v.replace(old, new)
        x[k] = v
    return x


def list_replace_value(l: list, old: str, new: str) -> list:
    x = []
    for e in l:
        if isinstance(e, list):
            e = list_replace_value(e, old, new)
        elif isinstance(e, dict):
            e = dict_replace_value(e, old, new)
        elif isinstance(e, str):
            e = e.replace(old, new)
        x.append(e)
    return x

# See input and output below
output = dict_replace_value(input, 'string', 'something')

Input:

input = {
    'key1': 'a string',
    'key2': 'another string',
    'key3': [
        'a string',
        'another string',
        [1, 2, 3],
        {
            'key1': 'a string',
            'key2': 'another string'
        }
    ],
    'key4': {
        'key1': 'a string',
        'key2': 'another string',
        'key3': [
            'a string',
            'another string',
            500,
            1000
        ]
    },
    'key5': {
        'key1': [
            {
                'key1': 'a string'
            }
        ]
    }
}

Output:

print(output)

{
   "key1":"a something",
   "key2":"another something",
   "key3":[
      "a something",
      "another something",
      [
         1,
         2,
         3
      ],
      {
         "key1":"a something",
         "key2":"another something"
      }
   ],
   "key4":{
      "key1":"a something",
      "key2":"another something",
      "key3":[
         "a something",
         "another something",
         500,
         1000
      ]
   },
   "key5":{
      "key1":[
         {
            "key1":"a something"
         }
      ]
   }
}

A general approach (assuming you don't know in advance which key of the dict is pointing to a list) would be to iterate over the dict and check the type of its values and then iterate again into each value if needed.

In your case, your dictionary may contain a list of dictionaries as values, so it is enough to check if a value is of type list, if so, iterate over the list and change the dicts you need.

It can be done recursively with a function like the following:

def f(d):
    for k,v in d.items():
        if k == 'current_value':
            d[k] = int(v)
        elif type(v) is list:
            for item in v:
                if type(item) is dict:
                    f(item)

>>> d = {'id': '10', 'datastreams': [{'current_value': '5'}, {'current_value': '4'}]}
>>> f(d)
>>> d
{'id': '10', 'datastreams': [{'current_value': 5}, {'current_value': 4}]}  

Can be done with list comprehension:

d['datastreams'] = [{'current_value': int(ds['current_value'])} if ('current_value' in ds) else ds for ds in d['datastreams']]

You can use ast.literal_eval to evaluate the underlying value for items with current_value key in the d['datastreams'] list. Then check whether the type is an int using isinstance for such values. Finally, type cast such values to int.

import ast
d = {'id': '10', 'datastreams': [{'current_value': '5'}, {'current_value': '4'}]}
for i in d['datastreams']:
    for k,v in i.items():
        if 'current_value' in k and isinstance(ast.literal_eval(v),int):
            i[k] = int(v)
#Output:
print(d)
{'id': '10', 'datastreams': [{'current_value': 5}, {'current_value': 4}]}

You could use this method which would loop through checks for current_value in list and change it to integer by passing the value through int() function:

for value in d.values():
    for element in value:
        if 'current_value' in element:
            element['current_value'] = int(element['current_value'])

Taking alec_djinn's solution little farther to handle also nested dicts:

def f(d):
    for k,v in d.items():
        if k == 'current_value':
            d[k] = int(v)
        elif type(v) is list:
            for item in v:
                if type(item) is dict:
                    f(item)
        if type(v) is dict:
            f(v)

Based on Cloudkollektiv 's excellent idea, I've refactored the code which basically can do the same as the original functions, but all in one. I've also added the possibility, instead of replacing an orignal string by a replacement string, to replace every value by fn(value) where fn is a function given as argument to original.

[EDIT] I've improved the code so it retains the initial object type, which comes in handy when dealing with structs like CommentedMap from ruamel.yaml
I've also added an option to replace all values by fn(key, value) for dicts where fn is a function given as argument to original. [/EDIT]

def replace_in_iterable(
    src: Union[dict, list],
    original: Union[str, Callable],
    replacement: Any = None,
    callable_wants_key: bool = False,
):
    """
    Recursive replace data in a struct

    Replaces every instance of string original with string replacement in a list/dict

    If original is a callable function, it will replace every value with original(value)
    If original is a callable function and callable_wants_key == True,
      it will replace every value with original(key, value) for dicts
      and with original(value) for any other data types
    """

    def _replace_in_iterable(key, _src):
        if isinstance(_src, dict) or isinstance(_src, list):
            _src = replace_in_iterable(_src, original, replacement, callable_wants_key)
        elif isinstance(original, Callable):
            if callable_wants_key:
                _src = original(key, _src)
            else:
                _src = original(_src)
        elif isinstance(_src, str) and isinstance(replacement, str):
            _src = _src.replace(original, replacement)
        else:
            _src = replacement
        return _src

    if isinstance(src, dict):
        for key, value in src.items():
            src[key] = _replace_in_iterable(key, value)
    elif isinstance(src, list):
        result = []
        for entry in src:
            result.append(_replace_in_iterable(None, entry))
        src = result
    else:
        src = _replace_in_iterable(None, src)
    return src

TL;DR: You can directly use this function as package via pip with:

pip install ofunctions.misc

Then use it with

from ofunctions.misc import replace_in_iterable

def test(string):
   return f"-{string}-"

output = replace_in_iterable(input, test)

Input

input = {
    'key1': 'a string',
    'key2': 'another string',
    'key3': [
        'a string',
        'another string',
        [1, 2, 3],
        {
            'key1': 'a string',
            'key2': 'another string'
        }
    ],
    'key4': {
        'key1': 'a string',
        'key2': 'another string',
        'key3': [
            'a string',
            'another string',
            500,
            1000
        ]
    },
    'key5': {
        'key1': [
            {
                'key1': 'a string'
            }
        ]
    }
}

Output

input = {
    'key1': '-a string-',
    'key2': '-another string-',
    'key3': [
        '-a string-',
        '-another string-',
        ['-1-', '-2-', '-3-'],
        {
            'key1': '-a string-',
            'key2': '-another string-'
        }
    ],
    'key4': {
        'key1': '-a string-',
        'key2': '-another string-',
        'key3': [
            '-a string-',
            '-another string-',
            '-500-',
            '-1000-'
        ]
    },
    'key5': {
        'key1': [
            {
                'key1': '-a string-'
            }
        ]
    }
}

Of course the original syntax via output = replace_in_iterable(input, "string", "something) still works.