stages-of-computation

import dask
from dask.distributed import Client
import dask.array as da

import numpy as np

%load_ext snakeviz

client = Client("tcp://172.20.104.205:8786")
client

Client

Client-dec853f0-a65e-11ef-80b7-acde48001122

    <tr>

        <td style="text-align: left;"><strong>Connection method:</strong> Direct</td>
        <td style="text-align: left;"></td>

    </tr>


        <tr>
            <td style="text-align: left;">
                <strong>Dashboard: </strong> <a href="http://172.20.104.205:8787/status" target="_blank">http://172.20.104.205:8787/status</a>
            </td>
            <td style="text-align: left;"></td>
        </tr>


    </table>




        <details>
        <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Scheduler Info</h3></summary>
        <div style="">
<div>
    <div style="width: 24px; height: 24px; background-color: #FFF7E5; border: 3px solid #FF6132; border-radius: 5px; position: absolute;"> </div>
    <div style="margin-left: 48px;">
        <h3 style="margin-bottom: 0px;">Scheduler</h3>
        <p style="color: #9D9D9D; margin-bottom: 0px;">Scheduler-8ffa2bc2-0cbb-48e2-b818-41edc5e5d4d8</p>
        <table style="width: 100%; text-align: left;">
            <tr>
                <td style="text-align: left;">
                    <strong>Comm:</strong> tcp://172.20.104.205:8786
                </td>
                <td style="text-align: left;">
                    <strong>Workers:</strong> 3
                </td>
            </tr>
            <tr>
                <td style="text-align: left;">
                    <strong>Dashboard:</strong> <a href="http://172.20.104.205:8787/status" target="_blank">http://172.20.104.205:8787/status</a>
                </td>
                <td style="text-align: left;">
                    <strong>Total threads:</strong> 36
                </td>
            </tr>
            <tr>
                <td style="text-align: left;">
                    <strong>Started:</strong> 2 minutes ago
                </td>
                <td style="text-align: left;">
                    <strong>Total memory:</strong> 48.00 GiB
                </td>
            </tr>
        </table>
    </div>
</div>

<details style="margin-left: 48px;">
    <summary style="margin-bottom: 20px;">
        <h3 style="display: inline;">Workers</h3>
    </summary>


    <div style="margin-bottom: 20px;">
        <div style="width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;"> </div>
        <div style="margin-left: 48px;">
        <details>
            <summary>
                <h4 style="margin-bottom: 0px; display: inline;">Worker: tcp://172.20.104.205:50218</h4>
            </summary>
            <table style="width: 100%; text-align: left;">
                <tr>
                    <td style="text-align: left;">
                        <strong>Comm: </strong> tcp://172.20.104.205:50218
                    </td>
                    <td style="text-align: left;">
                        <strong>Total threads: </strong> 12
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Dashboard: </strong> <a href="http://172.20.104.205:50219/status" target="_blank">http://172.20.104.205:50219/status</a>
                    </td>
                    <td style="text-align: left;">
                        <strong>Memory: </strong> 16.00 GiB
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Nanny: </strong> tcp://172.20.104.205:50216
                    </td>
                    <td style="text-align: left;"></td>
                </tr>
                <tr>
                    <td colspan="2" style="text-align: left;">
                        <strong>Local directory: </strong> /var/folders/7y/91xrb3bs5g96r0pvlvq04yx00000gn/T/dask-scratch-space/worker-3j6kie84
                    </td>
                </tr>




                <tr>
                    <td style="text-align: left;">
                        <strong>Tasks executing: </strong> 
                    </td>
                    <td style="text-align: left;">
                        <strong>Tasks in memory: </strong> 
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Tasks ready: </strong> 
                    </td>
                    <td style="text-align: left;">
                        <strong>Tasks in flight: </strong>
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>CPU usage:</strong> 3.0%
                    </td>
                    <td style="text-align: left;">
                        <strong>Last seen: </strong> Just now
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Memory usage: </strong> 117.25 MiB
                    </td>
                    <td style="text-align: left;">
                        <strong>Spilled bytes: </strong> 0 B
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Read bytes: </strong> 6.00 kiB
                    </td>
                    <td style="text-align: left;">
                        <strong>Write bytes: </strong> 6.00 kiB
                    </td>
                </tr>


            </table>
        </details>
        </div>
    </div>

    <div style="margin-bottom: 20px;">
        <div style="width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;"> </div>
        <div style="margin-left: 48px;">
        <details>
            <summary>
                <h4 style="margin-bottom: 0px; display: inline;">Worker: tcp://172.20.104.205:50228</h4>
            </summary>
            <table style="width: 100%; text-align: left;">
                <tr>
                    <td style="text-align: left;">
                        <strong>Comm: </strong> tcp://172.20.104.205:50228
                    </td>
                    <td style="text-align: left;">
                        <strong>Total threads: </strong> 12
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Dashboard: </strong> <a href="http://172.20.104.205:50229/status" target="_blank">http://172.20.104.205:50229/status</a>
                    </td>
                    <td style="text-align: left;">
                        <strong>Memory: </strong> 16.00 GiB
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Nanny: </strong> tcp://172.20.104.205:50226
                    </td>
                    <td style="text-align: left;"></td>
                </tr>
                <tr>
                    <td colspan="2" style="text-align: left;">
                        <strong>Local directory: </strong> /var/folders/7y/91xrb3bs5g96r0pvlvq04yx00000gn/T/dask-scratch-space/worker-iuwpvy_f
                    </td>
                </tr>




                <tr>
                    <td style="text-align: left;">
                        <strong>Tasks executing: </strong> 
                    </td>
                    <td style="text-align: left;">
                        <strong>Tasks in memory: </strong> 
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Tasks ready: </strong> 
                    </td>
                    <td style="text-align: left;">
                        <strong>Tasks in flight: </strong>
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>CPU usage:</strong> 2.9%
                    </td>
                    <td style="text-align: left;">
                        <strong>Last seen: </strong> Just now
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Memory usage: </strong> 118.91 MiB
                    </td>
                    <td style="text-align: left;">
                        <strong>Spilled bytes: </strong> 0 B
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Read bytes: </strong> 4.00 kiB
                    </td>
                    <td style="text-align: left;">
                        <strong>Write bytes: </strong> 4.00 kiB
                    </td>
                </tr>


            </table>
        </details>
        </div>
    </div>

    <div style="margin-bottom: 20px;">
        <div style="width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;"> </div>
        <div style="margin-left: 48px;">
        <details>
            <summary>
                <h4 style="margin-bottom: 0px; display: inline;">Worker: tcp://172.20.104.205:50235</h4>
            </summary>
            <table style="width: 100%; text-align: left;">
                <tr>
                    <td style="text-align: left;">
                        <strong>Comm: </strong> tcp://172.20.104.205:50235
                    </td>
                    <td style="text-align: left;">
                        <strong>Total threads: </strong> 12
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Dashboard: </strong> <a href="http://172.20.104.205:50238/status" target="_blank">http://172.20.104.205:50238/status</a>
                    </td>
                    <td style="text-align: left;">
                        <strong>Memory: </strong> 16.00 GiB
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Nanny: </strong> tcp://172.20.104.205:50233
                    </td>
                    <td style="text-align: left;"></td>
                </tr>
                <tr>
                    <td colspan="2" style="text-align: left;">
                        <strong>Local directory: </strong> /var/folders/7y/91xrb3bs5g96r0pvlvq04yx00000gn/T/dask-scratch-space/worker-t2wywt9q
                    </td>
                </tr>




                <tr>
                    <td style="text-align: left;">
                        <strong>Tasks executing: </strong> 
                    </td>
                    <td style="text-align: left;">
                        <strong>Tasks in memory: </strong> 
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Tasks ready: </strong> 
                    </td>
                    <td style="text-align: left;">
                        <strong>Tasks in flight: </strong>
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>CPU usage:</strong> 2.8%
                    </td>
                    <td style="text-align: left;">
                        <strong>Last seen: </strong> Just now
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Memory usage: </strong> 117.27 MiB
                    </td>
                    <td style="text-align: left;">
                        <strong>Spilled bytes: </strong> 0 B
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Read bytes: </strong> 4.00 kiB
                    </td>
                    <td style="text-align: left;">
                        <strong>Write bytes: </strong> 4.00 kiB
                    </td>
                </tr>


            </table>
        </details>
        </div>
    </div>


</details>

</div>

2024-11-19 20:48:16,286 - distributed.client - ERROR - Failed to reconnect to scheduler after 30.00 seconds, closing client

a = da.array(np.arange(10000*100).reshape(10000,100))
a = a.rechunk((1000, 10))
a

<tr> <th> Bytes </th> <td> 7.63 MiB </td> <td> 78.12 kiB </td> </tr> <tr> <th> Shape </th> <td> (10000, 100) </td> <td> (1000, 10) </td> </tr> <tr> <th> Dask graph </th> <td colspan="2"> 100 chunks in 2 graph layers </td> </tr> <tr> <th> Data type </th> <td colspan="2"> int64 numpy.ndarray </td> </tr> </tbody> </table> </td> <td> <svg width="75" height="170" style="stroke:rgb(0,0,0);stroke-width:1" > 100 10000 Array Chunk a.compute() array([[ 0, 1, 2, ..., 97, 98, 99], [ 100, 101, 102, ..., 197, 198, 199], [ 200, 201, 202, ..., 297, 298, 299], ..., [999700, 999701, 999702, ..., 999797, 999798, 999799], [999800, 999801, 999802, ..., 999897, 999898, 999899], [999900, 999901, 999902, ..., 999997, 999998, 999999]])