bitPhile

ordering

14 min read

import dask.array as da
import dask
from dask.distributed import Client, performance_report

x = da.zeros((12500, 10000), chunks=('10MB', -1))
x

                <tr>
                    <th> Bytes </th>
                    <td> 0.93 GiB </td>
                    <td> 9.54 MiB </td>
                </tr>

                <tr>
                    <th> Shape </th>
                    <td> (12500, 10000) </td>
                    <td> (125, 10000) </td>
                </tr>
                <tr>
                    <th> Dask graph </th>
                    <td colspan="2"> 100 chunks in 1 graph layer </td>
                </tr>
                <tr>
                    <th> Data type </th>
                    <td colspan="2"> float64 numpy.ndarray </td>
                </tr>
            </tbody>
        </table>
    </td>
    <td>
    <svg width="146" height="170" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 12500

Array Chunk 
da.to_zarr(x, "__pycache__/data/saved_x1.zarr", overwrite=True)
da.to_zarr(x, "__pycache__/data/saved_y1.zarr", overwrite=True)

da.to_zarr(x, "__pycache__/data/saved_x2.zarr", overwrite=True)
da.to_zarr(x, "__pycache__/data/saved_y2.zarr", overwrite=True)

x1 = da.from_zarr('__pycache__/data/saved_x1.zarr')
y1 = da.from_zarr('__pycache__/data/saved_x2.zarr')
x2 = da.from_zarr('__pycache__/data/saved_y1.zarr')
y2 = da.from_zarr('__pycache__/data/saved_y2.zarr')

def evaluate(x1, y1, x2, y2):
    u = da.stack([x1, y1])
    v = da.stack([x2, y2])
    
    components = [u, v, u ** 2 + v ** 2]
    return [
        abs(c[0] - c[1]).mean(axis=-1)
        for c in components
    ]

n = 125 * 4
results = evaluate(x1[:n], y1[:n], x2[:n], y2[:n])
results

[dask.array<mean_agg-aggregate, shape=(500,), dtype=float64, chunksize=(125,), chunktype=numpy.ndarray>,
 dask.array<mean_agg-aggregate, shape=(500,), dtype=float64, chunksize=(125,), chunktype=numpy.ndarray>,
 dask.array<mean_agg-aggregate, shape=(500,), dtype=float64, chunksize=(125,), chunktype=numpy.ndarray>]

dask.visualize(results,  optimize_graph=True, cmap="autumn", node_attr={"penwidth": "4"}, color='order')

png

results[0].dask

HighLevelGraph

HighLevelGraph with 13 layers and 242 keys from all layers. 

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer1: original-from-zarr</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        original-from-zarr-29046f8580ea540ea8b9402c9d9795c3
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">MaterializedLayer</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">1</td>
                </tr>


            </table>
        </td>
        <td>

        </td>
    </tr>
    </table>

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer2: from-zarr</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        from-zarr-29046f8580ea540ea8b9402c9d9795c3
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">Blockwise</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">100</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(12500, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(125, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">original-from-zarr-29046f8580ea540ea8b9402c9d9795c3</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="210" height="250" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 12500 

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer3: getitem</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        getitem-ed33f4257cc98c9d86ea274bb23f39ec
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">MaterializedLayer</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">4</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(500, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(125, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">from-zarr-29046f8580ea540ea8b9402c9d9795c3</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="250" height="107" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 500 

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer4: original-from-zarr</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        original-from-zarr-b86d06dc7c4736ceef56e0c0d0e5af4d
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">MaterializedLayer</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">1</td>
                </tr>


            </table>
        </td>
        <td>

        </td>
    </tr>
    </table>

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer5: from-zarr</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        from-zarr-b86d06dc7c4736ceef56e0c0d0e5af4d
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">Blockwise</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">100</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(12500, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(125, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">original-from-zarr-b86d06dc7c4736ceef56e0c0d0e5af4d</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="210" height="250" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 12500 

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer6: getitem</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        getitem-c8d738cbe21502c64bf9bd3a65eca0b5
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">MaterializedLayer</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">4</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(500, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(125, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">from-zarr-b86d06dc7c4736ceef56e0c0d0e5af4d</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="250" height="107" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 500 

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer7: stack</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        stack-576b40ac713fcb33a7513cd148008c3f
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">MaterializedLayer</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">8</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(2, 500, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(1, 125, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">getitem-ed33f4257cc98c9d86ea274bb23f39ec</td>
                    </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"></th>
                        <td style="text-align: left;">getitem-c8d738cbe21502c64bf9bd3a65eca0b5</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="284" height="132" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 500 2 

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer8: getitem</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        getitem-8a064bd748e2a143aa787cb3553dfdf9
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">MaterializedLayer</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">4</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(500, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(125, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">stack-576b40ac713fcb33a7513cd148008c3f</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="250" height="107" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 500 

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer9: getitem</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        getitem-e930bbed5998d3dd2d58a6aa337e457b
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">MaterializedLayer</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">4</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(500, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(125, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">stack-576b40ac713fcb33a7513cd148008c3f</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="250" height="107" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 500 

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer10: sub</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        sub-355c38cde8a95825d373f7b0914bd4c9
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">Blockwise</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">4</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(500, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(125, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">getitem-e930bbed5998d3dd2d58a6aa337e457b</td>
                    </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"></th>
                        <td style="text-align: left;">getitem-8a064bd748e2a143aa787cb3553dfdf9</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="250" height="107" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 500 

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer11: abs</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        abs-d8a1a780e1ba46de42c427217504d0d2
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">Blockwise</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">4</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(500, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(125, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">sub-355c38cde8a95825d373f7b0914bd4c9</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="250" height="107" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 500 

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer12: mean_chunk</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        mean_chunk-53c7b82e972b136fba07af296fb1d71a
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">Blockwise</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">4</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(500, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(125, 10000)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">abs-d8a1a780e1ba46de42c427217504d0d2</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="250" height="107" style="stroke:rgb(0,0,0);stroke-width:1" >

10000 500 

</details>

        <div style="">
<svg width="24" height="24" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg" style="position: absolute;">

    <circle cx="16" cy="16" r="14" fill="#8F8F8F" style="stroke: var(--jp-ui-font-color2, #1D1D1D);" stroke-width="2"/>

</svg>

<details style="margin-left: 32px;">
    <summary style="margin-bottom: 10px; margin-top: 10px;">
        <h4 style="display: inline;">Layer13: mean_agg-aggregate</h4>
    </summary>
    <p style="color: var(--jp-ui-font-color2, #5D5851); margin: -0.25em 0px 0px 0px;">
        mean_agg-aggregate-85b94be6a1c9bedbdda1c02171b37ba6
    </p>

    <table>
    <tr>
        <td>
            <table>

                <tr>
                    <th style="text-align: left; width: 150px;">layer_type</th>
                    <td style="text-align: left;">MaterializedLayer</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">is_materialized</th>
                    <td style="text-align: left;">True</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">number of outputs</th>
                    <td style="text-align: left;">4</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">shape</th>
                    <td style="text-align: left;">(500,)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">dtype</th>
                    <td style="text-align: left;">float64</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunksize</th>
                    <td style="text-align: left;">(125,)</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">type</th>
                    <td style="text-align: left;">dask.array.core.Array</td>
                </tr>

                <tr>
                    <th style="text-align: left; width: 150px;">chunk_type</th>
                    <td style="text-align: left;">numpy.ndarray</td>
                </tr>



                    <tr>
                        <th style="text-align: left; width: 150px;"> depends on </th>
                        <td style="text-align: left;">mean_chunk-53c7b82e972b136fba07af296fb1d71a</td>
                    </tr>


            </table>
        </td>
        <td>
            <svg width="250" height="92" style="stroke:rgb(0,0,0);stroke-width:1" >

500 1 

</details>

    </div>
</div>

client = Client("tcp://172.20.104.205:8786")
client

Client

Client-1d81e416-a8b3-11ef-a0f3-acde48001122

    <tr>

        <td style="text-align: left;"><strong>Connection method:</strong> Direct</td>
        <td style="text-align: left;"></td>

    </tr>


        <tr>
            <td style="text-align: left;">
                <strong>Dashboard: </strong> <a href="http://172.20.104.205:8787/status" target="_blank">http://172.20.104.205:8787/status</a>
            </td>
            <td style="text-align: left;"></td>
        </tr>


    </table>




        <details>
        <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Scheduler Info</h3></summary>
        <div style="">
<div>
    <div style="width: 24px; height: 24px; background-color: #FFF7E5; border: 3px solid #FF6132; border-radius: 5px; position: absolute;"> </div>
    <div style="margin-left: 48px;">
        <h3 style="margin-bottom: 0px;">Scheduler</h3>
        <p style="color: #9D9D9D; margin-bottom: 0px;">Scheduler-000827b1-3cb8-409d-b77e-594e9b6b470d</p>
        <table style="width: 100%; text-align: left;">
            <tr>
                <td style="text-align: left;">
                    <strong>Comm:</strong> tcp://172.20.104.205:8786
                </td>
                <td style="text-align: left;">
                    <strong>Workers:</strong> 2
                </td>
            </tr>
            <tr>
                <td style="text-align: left;">
                    <strong>Dashboard:</strong> <a href="http://172.20.104.205:8787/status" target="_blank">http://172.20.104.205:8787/status</a>
                </td>
                <td style="text-align: left;">
                    <strong>Total threads:</strong> 8
                </td>
            </tr>
            <tr>
                <td style="text-align: left;">
                    <strong>Started:</strong> Just now
                </td>
                <td style="text-align: left;">
                    <strong>Total memory:</strong> 8.00 GiB
                </td>
            </tr>
        </table>
    </div>
</div>

<details style="margin-left: 48px;">
    <summary style="margin-bottom: 20px;">
        <h3 style="display: inline;">Workers</h3>
    </summary>


    <div style="margin-bottom: 20px;">
        <div style="width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;"> </div>
        <div style="margin-left: 48px;">
        <details>
            <summary>
                <h4 style="margin-bottom: 0px; display: inline;">Worker: tcp://172.20.104.205:54988</h4>
            </summary>
            <table style="width: 100%; text-align: left;">
                <tr>
                    <td style="text-align: left;">
                        <strong>Comm: </strong> tcp://172.20.104.205:54988
                    </td>
                    <td style="text-align: left;">
                        <strong>Total threads: </strong> 4
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Dashboard: </strong> <a href="http://172.20.104.205:54989/status" target="_blank">http://172.20.104.205:54989/status</a>
                    </td>
                    <td style="text-align: left;">
                        <strong>Memory: </strong> 4.00 GiB
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Nanny: </strong> tcp://172.20.104.205:54986
                    </td>
                    <td style="text-align: left;"></td>
                </tr>
                <tr>
                    <td colspan="2" style="text-align: left;">
                        <strong>Local directory: </strong> /var/folders/7y/91xrb3bs5g96r0pvlvq04yx00000gn/T/dask-scratch-space/worker-vxz_961y
                    </td>
                </tr>




                <tr>
                    <td style="text-align: left;">
                        <strong>Tasks executing: </strong> 
                    </td>
                    <td style="text-align: left;">
                        <strong>Tasks in memory: </strong> 
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Tasks ready: </strong> 
                    </td>
                    <td style="text-align: left;">
                        <strong>Tasks in flight: </strong>
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>CPU usage:</strong> 3.1%
                    </td>
                    <td style="text-align: left;">
                        <strong>Last seen: </strong> Just now
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Memory usage: </strong> 119.92 MiB
                    </td>
                    <td style="text-align: left;">
                        <strong>Spilled bytes: </strong> 0 B
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Read bytes: </strong> 14.01 kiB
                    </td>
                    <td style="text-align: left;">
                        <strong>Write bytes: </strong> 6.00 kiB
                    </td>
                </tr>


            </table>
        </details>
        </div>
    </div>

    <div style="margin-bottom: 20px;">
        <div style="width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;"> </div>
        <div style="margin-left: 48px;">
        <details>
            <summary>
                <h4 style="margin-bottom: 0px; display: inline;">Worker: tcp://172.20.104.205:54996</h4>
            </summary>
            <table style="width: 100%; text-align: left;">
                <tr>
                    <td style="text-align: left;">
                        <strong>Comm: </strong> tcp://172.20.104.205:54996
                    </td>
                    <td style="text-align: left;">
                        <strong>Total threads: </strong> 4
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Dashboard: </strong> <a href="http://172.20.104.205:54997/status" target="_blank">http://172.20.104.205:54997/status</a>
                    </td>
                    <td style="text-align: left;">
                        <strong>Memory: </strong> 4.00 GiB
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Nanny: </strong> tcp://172.20.104.205:54994
                    </td>
                    <td style="text-align: left;"></td>
                </tr>
                <tr>
                    <td colspan="2" style="text-align: left;">
                        <strong>Local directory: </strong> /var/folders/7y/91xrb3bs5g96r0pvlvq04yx00000gn/T/dask-scratch-space/worker-5zofw279
                    </td>
                </tr>




                <tr>
                    <td style="text-align: left;">
                        <strong>Tasks executing: </strong> 
                    </td>
                    <td style="text-align: left;">
                        <strong>Tasks in memory: </strong> 
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Tasks ready: </strong> 
                    </td>
                    <td style="text-align: left;">
                        <strong>Tasks in flight: </strong>
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>CPU usage:</strong> 3.2%
                    </td>
                    <td style="text-align: left;">
                        <strong>Last seen: </strong> Just now
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Memory usage: </strong> 117.19 MiB
                    </td>
                    <td style="text-align: left;">
                        <strong>Spilled bytes: </strong> 0 B
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Read bytes: </strong> 14.02 kiB
                    </td>
                    <td style="text-align: left;">
                        <strong>Write bytes: </strong> 10.02 kiB
                    </td>
                </tr>


            </table>
        </details>
        </div>
    </div>


</details>

</div>

with performance_report(filename="dask-report.html"):
    dask.compute(*results)

dask.visualize(results[0])

png

Graph View