Collection expressions

Arrays, maps and structs.

ArrayValue

ArrayValue(self, arg)

Methods

Name	Description
concat	Concatenate this array with one or more arrays.
contains	Return whether the array contains `other`.
filter	Filter array elements using `predicate`.
flatten	Remove one level of nesting from an array expression.
index	Return the position of `other` in an array.
intersect	Intersect two arrays.
join	Join the elements of this array expression with `sep`.
length	Compute the length of an array.
map	Apply a callable `func` to each element of this array expression.
remove	Remove `other` from `self`.
repeat	Repeat this array `n` times.
sort	Sort the elements in an array.
union	Union two arrays.
unique	Return the unique values in an array.
unnest	Flatten an array into a column.
zip	Zip two or more arrays together.

concat

concat(other, *args)

Concatenate this array with one or more arrays.

Parameters

Name	Type	Description	Default
`other`	ArrayValue	Other array to concat with `self`	required
`args`	ArrayValue	Other arrays to concat with `self`	`()`

Returns

Type	Description
ArrayValue	`self` concatenated with `other` and `args`

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"a": [[7], [3], None]})
>>> t

┏━━━━━━━━━━━━━━┓
┃ a            ┃
┡━━━━━━━━━━━━━━┩
│ array<int64> │
├──────────────┤
│ [7]          │
│ [3]          │
│ NULL         │
└──────────────┘

>>> t.a.concat(t.a)

┏━━━━━━━━━━━━━━━┓
┃ ArrayConcat() ┃
┡━━━━━━━━━━━━━━━┩
│ array<int64>  │
├───────────────┤
│ [7, 7]        │
│ [3, 3]        │
│ NULL          │
└───────────────┘

>>> t.a.concat(ibis.literal([4], type="array<int64>"))

┏━━━━━━━━━━━━━━━┓
┃ ArrayConcat() ┃
┡━━━━━━━━━━━━━━━┩
│ array<int64>  │
├───────────────┤
│ [7, 4]        │
│ [3, 4]        │
│ [4]           │
└───────────────┘

concat is also available using the + operator

>>> [1] + t.a

┏━━━━━━━━━━━━━━━┓
┃ ArrayConcat() ┃
┡━━━━━━━━━━━━━━━┩
│ array<int64>  │
├───────────────┤
│ [1, 7]        │
│ [1, 3]        │
│ [1]           │
└───────────────┘

>>> t.a + [1]

┏━━━━━━━━━━━━━━━┓
┃ ArrayConcat() ┃
┡━━━━━━━━━━━━━━━┩
│ array<int64>  │
├───────────────┤
│ [7, 1]        │
│ [3, 1]        │
│ [1]           │
└───────────────┘

contains

contains(other)

Return whether the array contains other.

Parameters

Name	Type	Description	Default
`other`	`ir`.`Value`	Ibis expression to check for existence of in `self`	required

Returns

Type	Description
`BooleanValue`	Whether `other` is contained in `self`

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"arr": [[1], [], [42, 42], None]})
>>> t

┏━━━━━━━━━━━━━━┓
┃ arr          ┃
┡━━━━━━━━━━━━━━┩
│ array<int64> │
├──────────────┤
│ [1]          │
│ []           │
│ [42, 42]     │
│ NULL         │
└──────────────┘

>>> t.arr.contains(42)

┏━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayContains(arr, 42) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━┩
│ boolean                │
├────────────────────────┤
│ False                  │
│ False                  │
│ True                   │
│ NULL                   │
└────────────────────────┘

>>> t.arr.contains(None)

┏━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayContains(arr, None) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ boolean                  │
├──────────────────────────┤
│ NULL                     │
│ NULL                     │
│ NULL                     │
│ NULL                     │
└──────────────────────────┘

filter

filter(predicate)

Filter array elements using predicate.

Parameters

Name	Type	Description	Default
`predicate`	Callable[[`ir`.`Value`], bool \| `ir`.`BooleanValue`]	Function to use to filter array elements	required

Returns

Type	Description
ArrayValue	Array elements filtered using `predicate`

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"a": [[1, None, 2], [4], []]})
>>> t

┏━━━━━━━━━━━━━━━━━━━┓
┃ a                 ┃
┡━━━━━━━━━━━━━━━━━━━┩
│ array<int64>      │
├───────────────────┤
│ [1, None, ... +1] │
│ [4]               │
│ []                │
└───────────────────┘

>>> t.a.filter(lambda x: x > 1)

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayFilter(a, Greater(x, 1)) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<int64>                  │
├───────────────────────────────┤
│ [2]                           │
│ [4]                           │
│ []                            │
└───────────────────────────────┘

.filter() also supports more complex callables like functools.partial and lambdas with closures

>>> from functools import partial
>>> def gt(x, y):
...     return x > y
>>> gt1 = partial(gt, y=1)
>>> t.a.filter(gt1)

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayFilter(a, Greater(x, 1)) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<int64>                  │
├───────────────────────────────┤
│ [2]                           │
│ [4]                           │
│ []                            │
└───────────────────────────────┘

>>> y = 1
>>> t.a.filter(lambda x: x > y)

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayFilter(a, Greater(x, 1)) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<int64>                  │
├───────────────────────────────┤
│ [2]                           │
│ [4]                           │
│ []                            │
└───────────────────────────────┘

flatten

flatten()

Remove one level of nesting from an array expression.

Returns

Type	Description
ArrayValue	Flattened array expression

Examples

>>> import ibis
>>> import ibis.selectors as s
>>> from ibis import _
>>> ibis.options.interactive = True
>>> schema = {
...     "empty": "array<array<int>>",
...     "happy": "array<array<string>>",
...     "nulls_only": "array<array<struct<a: array<string>>>>",
...     "mixed_nulls": "array<array<string>>",
... }
>>> data = {
...     "empty": [[], [], []],
...     "happy": [[["abc"]], [["bcd"]], [["def"]]],
...     "nulls_only": [None, None, None],
...     "mixed_nulls": [[], None, [None]],
... }
>>> import pyarrow as pa
>>> t = ibis.memtable(
...     pa.Table.from_pydict(
...         data,
...         schema=ibis.schema(schema).to_pyarrow(),
...     )
... )
>>> t

┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓
┃ empty               ┃ happy                ┃ nulls_only ┃ mixed_nulls          ┃
┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩
│ array<array<int64>> │ array<array<string>> │ array<arr… │ array<array<string>> │
├─────────────────────┼──────────────────────┼────────────┼──────────────────────┤
│ []                  │ [[...]]              │ NULL       │ []                   │
│ []                  │ [[...]]              │ NULL       │ NULL                 │
│ []                  │ [[...]]              │ NULL       │ [None]               │
└─────────────────────┴──────────────────────┴────────────┴──────────────────────┘

>>> t.empty.flatten()

┏━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayFlatten(empty) ┃
┡━━━━━━━━━━━━━━━━━━━━━┩
│ array<int64>        │
├─────────────────────┤
│ []                  │
│ []                  │
│ []                  │
└─────────────────────┘

>>> t.happy.flatten()

┏━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayFlatten(happy) ┃
┡━━━━━━━━━━━━━━━━━━━━━┩
│ array<string>       │
├─────────────────────┤
│ ['abc']             │
│ ['bcd']             │
│ ['def']             │
└─────────────────────┘

>>> t.nulls_only.flatten()

┏━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayFlatten(nulls_only) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<struct<a: array<s… │
├──────────────────────────┤
│ NULL                     │
│ NULL                     │
│ NULL                     │
└──────────────────────────┘

>>> t.mixed_nulls.flatten()

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayFlatten(mixed_nulls) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<string>             │
├───────────────────────────┤
│ []                        │
│ NULL                      │
│ []                        │
└───────────────────────────┘

>>> t.select(s.across(s.all(), _.flatten()))

┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
┃ empty        ┃ happy         ┃ nulls_only ┃ mixed_nulls   ┃
┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
│ array<int64> │ array<string> │ array<str… │ array<string> │
├──────────────┼───────────────┼────────────┼───────────────┤
│ []           │ ['abc']       │ NULL       │ []            │
│ []           │ ['bcd']       │ NULL       │ NULL          │
│ []           │ ['def']       │ NULL       │ []            │
└──────────────┴───────────────┴────────────┴───────────────┘

index

index(other)

Return the position of other in an array.

Parameters

Name	Type	Description	Default
`other`	`ir`.`Value`	Ibis expression to existence of in `self`	required

Returns

Type	Description
`BooleanValue`	The position of `other` in `self`

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"arr": [[1], [], [42, 42], None]})
>>> t

┏━━━━━━━━━━━━━━┓
┃ arr          ┃
┡━━━━━━━━━━━━━━┩
│ array<int64> │
├──────────────┤
│ [1]          │
│ []           │
│ [42, 42]     │
│ NULL         │
└──────────────┘

>>> t.arr.index(42)

┏━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayPosition(arr, 42) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━┩
│ int64                  │
├────────────────────────┤
│                     -1 │
│                     -1 │
│                      0 │
│                   NULL │
└────────────────────────┘

>>> t.arr.index(800)

┏━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayPosition(arr, 800) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ int64                   │
├─────────────────────────┤
│                      -1 │
│                      -1 │
│                      -1 │
│                    NULL │
└─────────────────────────┘

>>> t.arr.index(None)

┏━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayPosition(arr, None) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ int64                    │
├──────────────────────────┤
│                     NULL │
│                     NULL │
│                     NULL │
│                     NULL │
└──────────────────────────┘

intersect

intersect(other)

Intersect two arrays.

Parameters

Name	Type	Description	Default
`other`	ArrayValue	Another array to intersect with `self`	required

Returns

Type	Description
ArrayValue	Intersected arrays

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"arr1": [[3, 2], [], None], "arr2": [[1, 3], [None], [5]]})
>>> t

┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓
┃ arr1         ┃ arr2         ┃
┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩
│ array<int64> │ array<int64> │
├──────────────┼──────────────┤
│ [3, 2]       │ [1, 3]       │
│ []           │ [None]       │
│ NULL         │ [5]          │
└──────────────┴──────────────┘

>>> t.arr1.intersect(t.arr2)

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayIntersect(arr1, arr2) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<int64>               │
├────────────────────────────┤
│ [3]                        │
│ []                         │
│ NULL                       │
└────────────────────────────┘

join

join(sep)

Join the elements of this array expression with sep.

Parameters

Name	Type	Description	Default
`sep`	str \| `ir`.`StringValue`	Separator to use for joining array elements	required

Returns

Type	Description
`StringValue`	Elements of `self` joined with `sep`

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"arr": [["a", "b", "c"], None, [], ["b", None]]})
>>> t

┏━━━━━━━━━━━━━━━━━━━━┓
┃ arr                ┃
┡━━━━━━━━━━━━━━━━━━━━┩
│ array<string>      │
├────────────────────┤
│ ['a', 'b', ... +1] │
│ NULL               │
│ []                 │
│ ['b', None]        │
└────────────────────┘

>>> t.arr.join("|")

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayStringJoin('|', arr) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ string                    │
├───────────────────────────┤
│ a|b|c                     │
│ NULL                      │
│ NULL                      │
│ b                         │
└───────────────────────────┘

length

length()

Compute the length of an array.

Returns

Type	Description
`IntegerValue`	The integer length of each element of `self`

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"a": [[7, 42], [3], None]})
>>> t

┏━━━━━━━━━━━━━━┓
┃ a            ┃
┡━━━━━━━━━━━━━━┩
│ array<int64> │
├──────────────┤
│ [7, 42]      │
│ [3]          │
│ NULL         │
└──────────────┘

>>> t.a.length()

┏━━━━━━━━━━━━━━━━┓
┃ ArrayLength(a) ┃
┡━━━━━━━━━━━━━━━━┩
│ int64          │
├────────────────┤
│              2 │
│              1 │
│           NULL │
└────────────────┘

map

map(func)

Apply a callable func to each element of this array expression.

Parameters

Name	Type	Description	Default
`func`	Callable[[`ir`.`Value`], `ir`.`Value`]	Function to apply to each element of this array	required

Returns

Type	Description
ArrayValue	`func` applied to every element of this array expression.

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"a": [[1, None, 2], [4], []]})
>>> t

┏━━━━━━━━━━━━━━━━━━━┓
┃ a                 ┃
┡━━━━━━━━━━━━━━━━━━━┩
│ array<int64>      │
├───────────────────┤
│ [1, None, ... +1] │
│ [4]               │
│ []                │
└───────────────────┘

>>> t.a.map(lambda x: (x + 100).cast("float"))

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayMap(a, Cast(Add(x, 100), float64)) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<float64>                          │
├─────────────────────────────────────────┤
│ [101.0, None, ... +1]                   │
│ [104.0]                                 │
│ []                                      │
└─────────────────────────────────────────┘

.map() also supports more complex callables like functools.partial and lambdas with closures

>>> from functools import partial
>>> def add(x, y):
...     return x + y
>>> add2 = partial(add, y=2)
>>> t.a.map(add2)

┏━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayMap(a, Add(x, 2)) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<int64>           │
├────────────────────────┤
│ [3, None, ... +1]      │
│ [6]                    │
│ []                     │
└────────────────────────┘

>>> y = 2
>>> t.a.map(lambda x: x + y)

┏━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayMap(a, Add(x, 2)) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<int64>           │
├────────────────────────┤
│ [3, None, ... +1]      │
│ [6]                    │
│ []                     │
└────────────────────────┘

remove

remove(other)

Remove other from self.

Parameters

Name	Type	Description	Default
`other`	`ir`.`Value`	Element to remove from `self`.	required

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"arr": [[3, 2], [], [42, 2], [2, 2], None]})
>>> t

┏━━━━━━━━━━━━━━┓
┃ arr          ┃
┡━━━━━━━━━━━━━━┩
│ array<int64> │
├──────────────┤
│ [3, 2]       │
│ []           │
│ [42, 2]      │
│ [2, 2]       │
│ NULL         │
└──────────────┘

>>> t.arr.remove(2)

┏━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayRemove(arr, 2) ┃
┡━━━━━━━━━━━━━━━━━━━━━┩
│ array<int64>        │
├─────────────────────┤
│ [3]                 │
│ []                  │
│ [42]                │
│ []                  │
│ NULL                │
└─────────────────────┘

repeat

repeat(n)

Repeat this array n times.

Parameters

Name	Type	Description	Default
`n`	int \| `ir`.`IntegerValue`	Number of times to repeat `self`.	required

Returns

Type	Description
ArrayValue	`self` repeated `n` times

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"a": [[7], [3], None]})
>>> t

┏━━━━━━━━━━━━━━┓
┃ a            ┃
┡━━━━━━━━━━━━━━┩
│ array<int64> │
├──────────────┤
│ [7]          │
│ [3]          │
│ NULL         │
└──────────────┘

>>> t.a.repeat(2)

┏━━━━━━━━━━━━━━━━━━━┓
┃ ArrayRepeat(a, 2) ┃
┡━━━━━━━━━━━━━━━━━━━┩
│ array<int64>      │
├───────────────────┤
│ [7, 7]            │
│ [3, 3]            │
│ []                │
└───────────────────┘

repeat is also available using the * operator

>>> 2 * t.a

┏━━━━━━━━━━━━━━━━━━━┓
┃ ArrayRepeat(a, 2) ┃
┡━━━━━━━━━━━━━━━━━━━┩
│ array<int64>      │
├───────────────────┤
│ [7, 7]            │
│ [3, 3]            │
│ []                │
└───────────────────┘

sort

sort()

Sort the elements in an array.

Returns

Type	Description
ArrayValue	Sorted values in an array

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"arr": [[3, 2], [], [42, 42], None]})
>>> t

┏━━━━━━━━━━━━━━┓
┃ arr          ┃
┡━━━━━━━━━━━━━━┩
│ array<int64> │
├──────────────┤
│ [3, 2]       │
│ []           │
│ [42, 42]     │
│ NULL         │
└──────────────┘

>>> t.arr.sort()

┏━━━━━━━━━━━━━━━━┓
┃ ArraySort(arr) ┃
┡━━━━━━━━━━━━━━━━┩
│ array<int64>   │
├────────────────┤
│ [2, 3]         │
│ []             │
│ [42, 42]       │
│ NULL           │
└────────────────┘

union

union(other)

Union two arrays.

Parameters

Name	Type	Description	Default
`other`	`ir`.`ArrayValue`	Another array to union with `self`	required

Returns

Type	Description
ArrayValue	Unioned arrays

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"arr1": [[3, 2], [], None], "arr2": [[1, 3], [None], [5]]})
>>> t

┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓
┃ arr1         ┃ arr2         ┃
┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩
│ array<int64> │ array<int64> │
├──────────────┼──────────────┤
│ [3, 2]       │ [1, 3]       │
│ []           │ [None]       │
│ NULL         │ [5]          │
└──────────────┴──────────────┘

>>> t.arr1.union(t.arr2)

┏━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayUnion(arr1, arr2) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<int64>           │
├────────────────────────┤
│ [1, 2, ... +1]         │
│ [None]                 │
│ [5]                    │
└────────────────────────┘

>>> t.arr1.union(t.arr2).contains(3)

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayContains(ArrayUnion(arr1, arr2), 3) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ boolean                                  │
├──────────────────────────────────────────┤
│ True                                     │
│ False                                    │
│ False                                    │
└──────────────────────────────────────────┘

unique

unique()

Return the unique values in an array.

Element ordering in array may not be retained.

Returns

Type	Description
ArrayValue	Unique values in an array

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"arr": [[1, 3, 3], [], [42, 42], None]})
>>> t

┏━━━━━━━━━━━━━━━━┓
┃ arr            ┃
┡━━━━━━━━━━━━━━━━┩
│ array<int64>   │
├────────────────┤
│ [1, 3, ... +1] │
│ []             │
│ [42, 42]       │
│ NULL           │
└────────────────┘

>>> t.arr.unique()

┏━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayDistinct(arr) ┃
┡━━━━━━━━━━━━━━━━━━━━┩
│ array<int64>       │
├────────────────────┤
│ [3, 1]             │
│ []                 │
│ [42]               │
│ NULL               │
└────────────────────┘

unnest

unnest()

Flatten an array into a column.

Rows with empty arrays are dropped in the output.

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"a": [[7, 42], [3, 3], None]})
>>> t

┏━━━━━━━━━━━━━━┓
┃ a            ┃
┡━━━━━━━━━━━━━━┩
│ array<int64> │
├──────────────┤
│ [7, 42]      │
│ [3, 3]       │
│ NULL         │
└──────────────┘

>>> t.a.unnest()

┏━━━━━━━┓
┃ a     ┃
┡━━━━━━━┩
│ int64 │
├───────┤
│     7 │
│    42 │
│     3 │
│     3 │
└───────┘

Returns

Type	Description
`ir`.`Value`	Unnested array

zip

zip(other, *others)

Zip two or more arrays together.

Parameters

Name	Type	Description	Default
`other`	ArrayValue	Another array to zip with `self`	required
`others`	ArrayValue	Additional arrays to zip with `self`	`()`

Returns

Type	Description
`Array`	Array of structs where each struct field is an element of each input array.

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"numbers": [[3, 2], [], None], "strings": [["a", "c"], None, ["e"]]})
>>> t

┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
┃ numbers      ┃ strings       ┃
┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
│ array<int64> │ array<string> │
├──────────────┼───────────────┤
│ [3, 2]       │ ['a', 'c']    │
│ []           │ NULL          │
│ NULL         │ ['e']         │
└──────────────┴───────────────┘

>>> expr = t.numbers.zip(t.strings)
>>> expr

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayZip()                           ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ array<struct<f1: int64, f2: string>> │
├──────────────────────────────────────┤
│ [{...}, {...}]                       │
│ []                                   │
│ [{...}]                              │
└──────────────────────────────────────┘

>>> expr.unnest()

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ArrayZip()                    ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ struct<f1: int64, f2: string> │
├───────────────────────────────┤
│ {'f1': 3, 'f2': 'a'}          │
│ {'f1': 2, 'f2': 'c'}          │
│ {'f1': None, 'f2': 'e'}       │
└───────────────────────────────┘

MapValue

MapValue(self, arg)

A map literal or column expression.

Can be constructed with ibis.map().

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> import pyarrow as pa
>>> tab = pa.table(
...     {
...         "m": pa.array(
...             [[("a", 1), ("b", 2)], [("a", 1)], None],
...             type=pa.map_(pa.utf8(), pa.int64()),
...         )
...     }
... )
>>> t = ibis.memtable(tab)
>>> t

┏━━━━━━━━━━━━━━━━━━━━━┓
┃ m                   ┃
┡━━━━━━━━━━━━━━━━━━━━━┩
│ map<!string, int64> │
├─────────────────────┤
│ {'a': 1, 'b': 2}    │
│ {'a': 1}            │
│ NULL                │
└─────────────────────┘

Can use [] to access values:

>>> t.m["a"]

┏━━━━━━━━━━━━━━━━━━━━━━┓
┃ MapGet(m, 'a', None) ┃
┡━━━━━━━━━━━━━━━━━━━━━━┩
│ int64                │
├──────────────────────┤
│                    1 │
│                    1 │
│                 NULL │
└──────────────────────┘

To provide default values, use get:

>>> t.m.get("b", 0)

┏━━━━━━━━━━━━━━━━━━━┓
┃ MapGet(m, 'b', 0) ┃
┡━━━━━━━━━━━━━━━━━━━┩
│ int64             │
├───────────────────┤
│                 2 │
│                 0 │
│                 0 │
└───────────────────┘

Methods

Name	Description
contains	Return whether the map contains `key`.
get	Return the value for `key` from `expr`.
keys	Extract the keys of a map.
length	Return the number of key-value pairs in the map.
values	Extract the values of a map.

contains

contains(key)

Return whether the map contains key.

Parameters

Name	Type	Description	Default
`key`	int \| str \| `ir`.`IntegerValue` \| `ir`.`StringValue`	Mapping key for which to check	required

Returns

Type	Description
`BooleanValue`	Boolean indicating the presence of `key` in the map expression

Examples

>>> import ibis
>>> import pyarrow as pa
>>> ibis.options.interactive = True
>>> tab = pa.table(
...     {
...         "m": pa.array(
...             [[("a", 1), ("b", 2)], [("a", 1)], None],
...             type=pa.map_(pa.utf8(), pa.int64()),
...         )
...     }
... )
>>> t = ibis.memtable(tab)
>>> t

┏━━━━━━━━━━━━━━━━━━━━━┓
┃ m                   ┃
┡━━━━━━━━━━━━━━━━━━━━━┩
│ map<!string, int64> │
├─────────────────────┤
│ {'a': 1, 'b': 2}    │
│ {'a': 1}            │
│ NULL                │
└─────────────────────┘

>>> t.m.contains("b")

┏━━━━━━━━━━━━━━━━━━━━━┓
┃ MapContains(m, 'b') ┃
┡━━━━━━━━━━━━━━━━━━━━━┩
│ boolean             │
├─────────────────────┤
│ True                │
│ False               │
│ False               │
└─────────────────────┘

get

get(key, default=None)

Return the value for key from expr.

Return default if key is not in the map.

Parameters

Name	Type	Description	Default
`key`	`ir`.`Value`	Expression to use for key	required
`default`	`ir`.`Value` \| None	Expression to return if `key` is not a key in `expr`	`None`

Returns

Type	Description
Value	The element type of `self`

Examples

>>> import ibis
>>> import pyarrow as pa
>>> ibis.options.interactive = True
>>> tab = pa.table(
...     {
...         "m": pa.array(
...             [[("a", 1), ("b", 2)], [("a", 1)], None],
...             type=pa.map_(pa.utf8(), pa.int64()),
...         )
...     }
... )
>>> t = ibis.memtable(tab)
>>> t

┏━━━━━━━━━━━━━━━━━━━━━┓
┃ m                   ┃
┡━━━━━━━━━━━━━━━━━━━━━┩
│ map<!string, int64> │
├─────────────────────┤
│ {'a': 1, 'b': 2}    │
│ {'a': 1}            │
│ NULL                │
└─────────────────────┘

>>> t.m.get("a")

┏━━━━━━━━━━━━━━━━━━━━━━┓
┃ MapGet(m, 'a', None) ┃
┡━━━━━━━━━━━━━━━━━━━━━━┩
│ int64                │
├──────────────────────┤
│                    1 │
│                    1 │
│                 NULL │
└──────────────────────┘

>>> t.m.get("b")

┏━━━━━━━━━━━━━━━━━━━━━━┓
┃ MapGet(m, 'b', None) ┃
┡━━━━━━━━━━━━━━━━━━━━━━┩
│ int64                │
├──────────────────────┤
│                    2 │
│                 NULL │
│                 NULL │
└──────────────────────┘

>>> t.m.get("b", 0)

┏━━━━━━━━━━━━━━━━━━━┓
┃ MapGet(m, 'b', 0) ┃
┡━━━━━━━━━━━━━━━━━━━┩
│ int64             │
├───────────────────┤
│                 2 │
│                 0 │
│                 0 │
└───────────────────┘

keys

keys()

Extract the keys of a map.

Returns

Type	Description
`ArrayValue`	The keys of `self`

Examples

>>> import ibis
>>> import pyarrow as pa
>>> ibis.options.interactive = True
>>> tab = pa.table(
...     {
...         "m": pa.array(
...             [[("a", 1), ("b", 2)], [("a", 1)], None],
...             type=pa.map_(pa.utf8(), pa.int64()),
...         )
...     }
... )
>>> t = ibis.memtable(tab)
>>> t

┏━━━━━━━━━━━━━━━━━━━━━┓
┃ m                   ┃
┡━━━━━━━━━━━━━━━━━━━━━┩
│ map<!string, int64> │
├─────────────────────┤
│ {'a': 1, 'b': 2}    │
│ {'a': 1}            │
│ NULL                │
└─────────────────────┘

>>> t.m.keys()

┏━━━━━━━━━━━━━━━━┓
┃ MapKeys(m)     ┃
┡━━━━━━━━━━━━━━━━┩
│ array<!string> │
├────────────────┤
│ ['a', 'b']     │
│ ['a']          │
│ NULL           │
└────────────────┘

length

length()

Return the number of key-value pairs in the map.

Returns

Type	Description
`IntegerValue`	The number of elements in `self`

Examples

>>> import ibis
>>> import pyarrow as pa
>>> ibis.options.interactive = True
>>> tab = pa.table(
...     {
...         "m": pa.array(
...             [[("a", 1), ("b", 2)], [("a", 1)], None],
...             type=pa.map_(pa.utf8(), pa.int64()),
...         )
...     }
... )
>>> t = ibis.memtable(tab)
>>> t

┏━━━━━━━━━━━━━━━━━━━━━┓
┃ m                   ┃
┡━━━━━━━━━━━━━━━━━━━━━┩
│ map<!string, int64> │
├─────────────────────┤
│ {'a': 1, 'b': 2}    │
│ {'a': 1}            │
│ NULL                │
└─────────────────────┘

>>> t.m.length()

┏━━━━━━━━━━━━━━┓
┃ MapLength(m) ┃
┡━━━━━━━━━━━━━━┩
│ int64        │
├──────────────┤
│            2 │
│            1 │
│         NULL │
└──────────────┘

values

values()

Extract the values of a map.

Returns

Type	Description
`ArrayValue`	The values of `self`

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> m = ibis.map({"a": 1, "b": 2})
>>> m.values()

[1, 2]

StructValue

StructValue(self, arg)

A struct literal or column.

Can be constructed with ibis.struct().

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"s": [{"a": 1, "b": "foo"}, {"a": 3, "b": None}, None]})
>>> t

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ s                           ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ struct<a: int64, b: string> │
├─────────────────────────────┤
│ {'a': 1, 'b': 'foo'}        │
│ {'a': 3, 'b': None}         │
│ NULL                        │
└─────────────────────────────┘

Can use either . or [] to access fields:

>>> t.s.a

┏━━━━━━━┓
┃ a     ┃
┡━━━━━━━┩
│ int64 │
├───────┤
│     1 │
│     3 │
│  NULL │
└───────┘

>>> t.s["a"]

┏━━━━━━━┓
┃ a     ┃
┡━━━━━━━┩
│ int64 │
├───────┤
│     1 │
│     3 │
│  NULL │
└───────┘

Attributes

Name	Description
fields	Return a mapping from field name to field type of the struct.
names	Return the field names of the struct.
types	Return the field types of the struct.

Methods

Name	Description
destructure	Destructure a `StructValue` into the corresponding struct fields.
lift	Project the fields of `self` into a table.

destructure

destructure()

Destructure a StructValue into the corresponding struct fields.

When assigned, a destruct value will be destructured and assigned to multiple columns.

Returns

Type	Description
list[`AnyValue`]	Value expressions corresponding to the struct fields.

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"s": [{"a": 1, "b": "foo"}, {"a": 3, "b": None}, None]})
>>> t

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ s                           ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ struct<a: int64, b: string> │
├─────────────────────────────┤
│ {'a': 1, 'b': 'foo'}        │
│ {'a': 3, 'b': None}         │
│ NULL                        │
└─────────────────────────────┘

>>> a, b = t.s.destructure()
>>> a

┏━━━━━━━┓
┃ a     ┃
┡━━━━━━━┩
│ int64 │
├───────┤
│     1 │
│     3 │
│  NULL │
└───────┘

>>> b

┏━━━━━━━━┓
┃ b      ┃
┡━━━━━━━━┩
│ string │
├────────┤
│ foo    │
│ NULL   │
│ NULL   │
└────────┘

lift

lift()

Project the fields of self into a table.

This method is useful when analyzing data that has deeply nested structs or arrays of structs. lift can be chained to avoid repeating column names and table references.

Returns

Type	Description
`Table`	A projection with this struct expression’s fields.

Examples

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable(
...     {
...         "pos": [
...             {"lat": 10.1, "lon": 30.3},
...             {"lat": 10.2, "lon": 30.2},
...             {"lat": 10.3, "lon": 30.1},
...         ]
...     }
... )
>>> t

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ pos                                ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ struct<lat: float64, lon: float64> │
├────────────────────────────────────┤
│ {'lat': 10.1, 'lon': 30.3}         │
│ {'lat': 10.2, 'lon': 30.2}         │
│ {'lat': 10.3, 'lon': 30.1}         │
└────────────────────────────────────┘

>>> t.pos.lift()

┏━━━━━━━━━┳━━━━━━━━━┓
┃ lat     ┃ lon     ┃
┡━━━━━━━━━╇━━━━━━━━━┩
│ float64 │ float64 │
├─────────┼─────────┤
│    10.1 │    30.3 │
│    10.2 │    30.2 │
│    10.3 │    30.1 │
└─────────┴─────────┘

array

ibis.array(values)

Create an array expression.

Parameters

Name	Type	Description	Default
`values`	Iterable[`V`]	An iterable of Ibis expressions or a list of Python literals	required

Returns

Type	Description
ArrayValue

Examples

Create an array from scalar values

>>> import ibis
>>> ibis.options.interactive = True
>>> ibis.array([1.0, None])

[1.0, None]

Create an array from column and scalar expressions

>>> t = ibis.memtable({"a": [1, 2, 3], "b": [4, 5, 6]})
>>> ibis.array([t.a, 42, ibis.literal(None)])

┏━━━━━━━━━━━━━━━━━┓
┃ Array()         ┃
┡━━━━━━━━━━━━━━━━━┩
│ array<int64>    │
├─────────────────┤
│ [1, 42, ... +1] │
│ [2, 42, ... +1] │
│ [3, 42, ... +1] │
└─────────────────┘

>>> ibis.array([t.a, 42 + ibis.literal(5)])

┏━━━━━━━━━━━━━━┓
┃ Array()      ┃
┡━━━━━━━━━━━━━━┩
│ array<int64> │
├──────────────┤
│ [1, 47]      │
│ [2, 47]      │
│ [3, 47]      │
└──────────────┘

map

ibis.map(keys, values=None)

Create a map container object.

If the keys and values are Python literals, then the output will be a MapScalar. If the keys and values are expressions (ArrayColumn), then the the output will be a MapColumn.

Parameters

Name	Type	Description	Default
`keys`	Iterable[Any] \| Mapping[Any, Any] \| `ArrayColumn`	Keys of the map or `Mapping`. If `keys` is a `Mapping`, `values` must be `None`.	required
`values`	Iterable[Any] \| `ArrayColumn` \| None	Values of the map or `None`. If `None`, the `keys` argument must be a `Mapping`.	`None`

Returns

Type	Description
MapValue	An expression representing either a map column or literal (associative array with key/value pairs of fixed types)

Examples

Create a map literal from a dict with the type inferred

>>> import ibis
>>> ibis.options.interactive = True
>>> ibis.map(dict(a=1, b=2))

{'a': 1, 'b': 2}

Create a new map column from columns with keys and values

>>> import ibis
>>> ibis.options.interactive = True
>>> t = ibis.memtable({"keys": [["a", "b"], ["b"]], "values": [[1, 2], [3]]})
>>> t

┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓
┃ keys          ┃ values       ┃
┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩
│ array<string> │ array<int64> │
├───────────────┼──────────────┤
│ ['a', 'b']    │ [1, 2]       │
│ ['b']         │ [3]          │
└───────────────┴──────────────┘

>>> ibis.map(t.keys, t.values)

┏━━━━━━━━━━━━━━━━━━━━┓
┃ Map(keys, values)  ┃
┡━━━━━━━━━━━━━━━━━━━━┩
│ map<string, int64> │
├────────────────────┤
│ {'a': 1, 'b': 2}   │
│ {'b': 3}           │
└────────────────────┘

struct

ibis.struct(value, type=None)

Create a struct expression.

If the input expressions are all column expressions, then the output will be a StructColumn.

If the input expressions are Python literals, then the output will be a StructScalar.

Parameters

Name	Type	Description	Default
`value`	Iterable[tuple[str, `V`]] \| Mapping[str, `V`]	The underlying data for literal struct value or a pairs of field names and column expressions.	required
`type`	str \| `dt`.`DataType` \| None	An instance of `ibis.expr.datatypes.DataType` or a string indicating the ibis type of `value`. This is only used if all of the input values are literals.	`None`

Returns

Type	Description
StructValue	An expression representing a literal or column struct (compound type with fields of fixed types)

Examples

Create a struct literal from a dict with the type inferred

>>> import ibis
>>> t = ibis.struct(dict(a=1, b="foo"))

Create a struct literal from a dict with a specified type

>>> t = ibis.struct(dict(a=1, b="foo"), type="struct<a: float, b: string>")

Specify a specific type for the struct literal

>>> t = ibis.struct(dict(a=1, b=40), type="struct<a: float, b: int32>")

Create a struct array from multiple arrays

>>> ibis.options.interactive = True
>>> t = ibis.memtable({"a": [1, 2, 3], "b": ["foo", "bar", "baz"]})
>>> ibis.struct([("a", t.a), ("b", t.b)])

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ StructColumn()              ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ struct<a: int64, b: string> │
├─────────────────────────────┤
│ {'a': 1, 'b': 'foo'}        │
│ {'a': 2, 'b': 'bar'}        │
│ {'a': 3, 'b': 'baz'}        │
└─────────────────────────────┘

Create a struct array from columns and literals

>>> ibis.struct([("a", t.a), ("b", "foo")])

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ StructColumn()              ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ struct<a: int64, b: string> │
├─────────────────────────────┤
│ {'a': 1, 'b': 'foo'}        │
│ {'a': 2, 'b': 'foo'}        │
│ {'a': 3, 'b': 'foo'}        │
└─────────────────────────────┘

ArrayValue

Methods

concat

Parameters

Returns

Examples

contains

Parameters

Returns

Examples

filter

Parameters

Returns

Examples

flatten

Returns

Examples

index

Parameters

Returns

Examples

intersect

Parameters

Returns

Examples

join

Parameters

Returns

Examples

See Also

length

Returns

Examples

map

Parameters

Returns

Examples

remove

Parameters

Examples

repeat

Parameters

Returns

Examples

sort

Returns

Examples

union

Parameters

Returns

Examples

unique

Returns

Examples

unnest

Examples

Returns

zip

Parameters

Returns

Examples

MapValue

Examples

Methods

contains

Parameters

Returns

Examples

get

Parameters

Returns

Examples

keys

Returns

Examples

length

Returns

Examples

values

Returns