bitPhile

debugging-in-dask

9 min read

from dask.distributed import Client, print
client = Client()
client

Client

Client-859821fa-9297-11ef-8a5c-acde48001122

    <tr>

        <td style="text-align: left;"><strong>Connection method:</strong> Cluster object</td>
        <td style="text-align: left;"><strong>Cluster type:</strong> distributed.LocalCluster</td>

    </tr>


        <tr>
            <td style="text-align: left;">
                <strong>Dashboard: </strong> <a href="http://127.0.0.1:8787/status" target="_blank">http://127.0.0.1:8787/status</a>
            </td>
            <td style="text-align: left;"></td>
        </tr>


    </table>




        <details>
        <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Cluster Info</h3></summary>
        <div class="jp-RenderedHTMLCommon jp-RenderedHTML jp-mod-trusted jp-OutputArea-output">
<div style="width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;">
</div>
<div style="margin-left: 48px;">
    <h3 style="margin-bottom: 0px; margin-top: 0px;">LocalCluster</h3>
    <p style="color: #9D9D9D; margin-bottom: 0px;">cb8eff24</p>
    <table style="width: 100%; text-align: left;">
        <tr>
            <td style="text-align: left;">
                <strong>Dashboard:</strong> <a href="http://127.0.0.1:8787/status" target="_blank">http://127.0.0.1:8787/status</a>
            </td>
            <td style="text-align: left;">
                <strong>Workers:</strong> 4
            </td>
        </tr>
        <tr>
            <td style="text-align: left;">
                <strong>Total threads:</strong> 12
            </td>
            <td style="text-align: left;">
                <strong>Total memory:</strong> 16.00 GiB
            </td>
        </tr>

        <tr>
<td style="text-align: left;"><strong>Status:</strong> running</td>
<td style="text-align: left;"><strong>Using processes:</strong> True</td>

    </table>

    <details>
        <summary style="margin-bottom: 20px;">
            <h3 style="display: inline;">Scheduler Info</h3>
        </summary>

        <div style="">
<div>
    <div style="width: 24px; height: 24px; background-color: #FFF7E5; border: 3px solid #FF6132; border-radius: 5px; position: absolute;"> </div>
    <div style="margin-left: 48px;">
        <h3 style="margin-bottom: 0px;">Scheduler</h3>
        <p style="color: #9D9D9D; margin-bottom: 0px;">Scheduler-3866a1d7-7ee1-4fb5-8dca-7d15316c288d</p>
        <table style="width: 100%; text-align: left;">
            <tr>
                <td style="text-align: left;">
                    <strong>Comm:</strong> tcp://127.0.0.1:63770
                </td>
                <td style="text-align: left;">
                    <strong>Workers:</strong> 4
                </td>
            </tr>
            <tr>
                <td style="text-align: left;">
                    <strong>Dashboard:</strong> <a href="http://127.0.0.1:8787/status" target="_blank">http://127.0.0.1:8787/status</a>
                </td>
                <td style="text-align: left;">
                    <strong>Total threads:</strong> 12
                </td>
            </tr>
            <tr>
                <td style="text-align: left;">
                    <strong>Started:</strong> Just now
                </td>
                <td style="text-align: left;">
                    <strong>Total memory:</strong> 16.00 GiB
                </td>
            </tr>
        </table>
    </div>
</div>

<details style="margin-left: 48px;">
    <summary style="margin-bottom: 20px;">
        <h3 style="display: inline;">Workers</h3>
    </summary>


    <div style="margin-bottom: 20px;">
        <div style="width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;"> </div>
        <div style="margin-left: 48px;">
        <details>
            <summary>
                <h4 style="margin-bottom: 0px; display: inline;">Worker: 0</h4>
            </summary>
            <table style="width: 100%; text-align: left;">
                <tr>
                    <td style="text-align: left;">
                        <strong>Comm: </strong> tcp://127.0.0.1:63781
                    </td>
                    <td style="text-align: left;">
                        <strong>Total threads: </strong> 3
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Dashboard: </strong> <a href="http://127.0.0.1:63785/status" target="_blank">http://127.0.0.1:63785/status</a>
                    </td>
                    <td style="text-align: left;">
                        <strong>Memory: </strong> 4.00 GiB
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Nanny: </strong> tcp://127.0.0.1:63773
                    </td>
                    <td style="text-align: left;"></td>
                </tr>
                <tr>
                    <td colspan="2" style="text-align: left;">
                        <strong>Local directory: </strong> /var/folders/7y/91xrb3bs5g96r0pvlvq04yx00000gn/T/dask-scratch-space/worker-2jsow2af
                    </td>
                </tr>





            </table>
        </details>
        </div>
    </div>

    <div style="margin-bottom: 20px;">
        <div style="width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;"> </div>
        <div style="margin-left: 48px;">
        <details>
            <summary>
                <h4 style="margin-bottom: 0px; display: inline;">Worker: 1</h4>
            </summary>
            <table style="width: 100%; text-align: left;">
                <tr>
                    <td style="text-align: left;">
                        <strong>Comm: </strong> tcp://127.0.0.1:63782
                    </td>
                    <td style="text-align: left;">
                        <strong>Total threads: </strong> 3
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Dashboard: </strong> <a href="http://127.0.0.1:63787/status" target="_blank">http://127.0.0.1:63787/status</a>
                    </td>
                    <td style="text-align: left;">
                        <strong>Memory: </strong> 4.00 GiB
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Nanny: </strong> tcp://127.0.0.1:63774
                    </td>
                    <td style="text-align: left;"></td>
                </tr>
                <tr>
                    <td colspan="2" style="text-align: left;">
                        <strong>Local directory: </strong> /var/folders/7y/91xrb3bs5g96r0pvlvq04yx00000gn/T/dask-scratch-space/worker-qqzq4_hi
                    </td>
                </tr>





            </table>
        </details>
        </div>
    </div>

    <div style="margin-bottom: 20px;">
        <div style="width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;"> </div>
        <div style="margin-left: 48px;">
        <details>
            <summary>
                <h4 style="margin-bottom: 0px; display: inline;">Worker: 2</h4>
            </summary>
            <table style="width: 100%; text-align: left;">
                <tr>
                    <td style="text-align: left;">
                        <strong>Comm: </strong> tcp://127.0.0.1:63783
                    </td>
                    <td style="text-align: left;">
                        <strong>Total threads: </strong> 3
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Dashboard: </strong> <a href="http://127.0.0.1:63789/status" target="_blank">http://127.0.0.1:63789/status</a>
                    </td>
                    <td style="text-align: left;">
                        <strong>Memory: </strong> 4.00 GiB
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Nanny: </strong> tcp://127.0.0.1:63775
                    </td>
                    <td style="text-align: left;"></td>
                </tr>
                <tr>
                    <td colspan="2" style="text-align: left;">
                        <strong>Local directory: </strong> /var/folders/7y/91xrb3bs5g96r0pvlvq04yx00000gn/T/dask-scratch-space/worker-oi153djn
                    </td>
                </tr>





            </table>
        </details>
        </div>
    </div>

    <div style="margin-bottom: 20px;">
        <div style="width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;"> </div>
        <div style="margin-left: 48px;">
        <details>
            <summary>
                <h4 style="margin-bottom: 0px; display: inline;">Worker: 3</h4>
            </summary>
            <table style="width: 100%; text-align: left;">
                <tr>
                    <td style="text-align: left;">
                        <strong>Comm: </strong> tcp://127.0.0.1:63784
                    </td>
                    <td style="text-align: left;">
                        <strong>Total threads: </strong> 3
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Dashboard: </strong> <a href="http://127.0.0.1:63791/status" target="_blank">http://127.0.0.1:63791/status</a>
                    </td>
                    <td style="text-align: left;">
                        <strong>Memory: </strong> 4.00 GiB
                    </td>
                </tr>
                <tr>
                    <td style="text-align: left;">
                        <strong>Nanny: </strong> tcp://127.0.0.1:63776
                    </td>
                    <td style="text-align: left;"></td>
                </tr>
                <tr>
                    <td colspan="2" style="text-align: left;">
                        <strong>Local directory: </strong> /var/folders/7y/91xrb3bs5g96r0pvlvq04yx00000gn/T/dask-scratch-space/worker-48liq_9i
                    </td>
                </tr>





            </table>
        </details>
        </div>
    </div>


</details>

    </details>
</div>

</div>

performing some distributed task
performing some distributed task
performed some distributed task
performing some distributed task
performed some distributed task
performing some distributed task
performed some distributed task
performing some distributed task
performed some distributed task

def task():
    print("performing some distributed task")
    import numpy as np
    result = np.arange(1000000).reshape((1000, 1000)).max(axis=1)
    print("performed some distributed task")
    return result
import dask
t1 = dask.delayed(task)()
t1
Delayed('task-4e6869f6-47b8-4e4c-8848-116d9d720b29')

dask.compute(t1)
(array([   999,   1999,   2999,   3999,   4999,   5999,   6999,   7999,
          8999,   9999,  10999,  11999,  12999,  13999,  14999,  15999,
         16999,  17999,  18999,  19999,  20999,  21999,  22999,  23999,
         24999,  25999,  26999,  27999,  28999,  29999,  30999,  31999,
         32999,  33999,  34999,  35999,  36999,  37999,  38999,  39999,
         40999,  41999,  42999,  43999,  44999,  45999,  46999,  47999,
         48999,  49999,  50999,  51999,  52999,  53999,  54999,  55999,
         56999,  57999,  58999,  59999,  60999,  61999,  62999,  63999,
         64999,  65999,  66999,  67999,  68999,  69999,  70999,  71999,
         72999,  73999,  74999,  75999,  76999,  77999,  78999,  79999,
         80999,  81999,  82999,  83999,  84999,  85999,  86999,  87999,
         88999,  89999,  90999,  91999,  92999,  93999,  94999,  95999,
         96999,  97999,  98999,  99999, 100999, 101999, 102999, 103999,
        104999, 105999, 106999, 107999, 108999, 109999, 110999, 111999,
        112999, 113999, 114999, 115999, 116999, 117999, 118999, 119999,
        120999, 121999, 122999, 123999, 124999, 125999, 126999, 127999,
        128999, 129999, 130999, 131999, 132999, 133999, 134999, 135999,
        136999, 137999, 138999, 139999, 140999, 141999, 142999, 143999,
        144999, 145999, 146999, 147999, 148999, 149999, 150999, 151999,
        152999, 153999, 154999, 155999, 156999, 157999, 158999, 159999,
        160999, 161999, 162999, 163999, 164999, 165999, 166999, 167999,
        168999, 169999, 170999, 171999, 172999, 173999, 174999, 175999,
        176999, 177999, 178999, 179999, 180999, 181999, 182999, 183999,
        184999, 185999, 186999, 187999, 188999, 189999, 190999, 191999,
        192999, 193999, 194999, 195999, 196999, 197999, 198999, 199999,
        200999, 201999, 202999, 203999, 204999, 205999, 206999, 207999,
        208999, 209999, 210999, 211999, 212999, 213999, 214999, 215999,
        216999, 217999, 218999, 219999, 220999, 221999, 222999, 223999,
        224999, 225999, 226999, 227999, 228999, 229999, 230999, 231999,
        232999, 233999, 234999, 235999, 236999, 237999, 238999, 239999,
        240999, 241999, 242999, 243999, 244999, 245999, 246999, 247999,
        248999, 249999, 250999, 251999, 252999, 253999, 254999, 255999,
        256999, 257999, 258999, 259999, 260999, 261999, 262999, 263999,
        264999, 265999, 266999, 267999, 268999, 269999, 270999, 271999,
        272999, 273999, 274999, 275999, 276999, 277999, 278999, 279999,
        280999, 281999, 282999, 283999, 284999, 285999, 286999, 287999,
        288999, 289999, 290999, 291999, 292999, 293999, 294999, 295999,
        296999, 297999, 298999, 299999, 300999, 301999, 302999, 303999,
        304999, 305999, 306999, 307999, 308999, 309999, 310999, 311999,
        312999, 313999, 314999, 315999, 316999, 317999, 318999, 319999,
        320999, 321999, 322999, 323999, 324999, 325999, 326999, 327999,
        328999, 329999, 330999, 331999, 332999, 333999, 334999, 335999,
        336999, 337999, 338999, 339999, 340999, 341999, 342999, 343999,
        344999, 345999, 346999, 347999, 348999, 349999, 350999, 351999,
        352999, 353999, 354999, 355999, 356999, 357999, 358999, 359999,
        360999, 361999, 362999, 363999, 364999, 365999, 366999, 367999,
        368999, 369999, 370999, 371999, 372999, 373999, 374999, 375999,
        376999, 377999, 378999, 379999, 380999, 381999, 382999, 383999,
        384999, 385999, 386999, 387999, 388999, 389999, 390999, 391999,
        392999, 393999, 394999, 395999, 396999, 397999, 398999, 399999,
        400999, 401999, 402999, 403999, 404999, 405999, 406999, 407999,
        408999, 409999, 410999, 411999, 412999, 413999, 414999, 415999,
        416999, 417999, 418999, 419999, 420999, 421999, 422999, 423999,
        424999, 425999, 426999, 427999, 428999, 429999, 430999, 431999,
        432999, 433999, 434999, 435999, 436999, 437999, 438999, 439999,
        440999, 441999, 442999, 443999, 444999, 445999, 446999, 447999,
        448999, 449999, 450999, 451999, 452999, 453999, 454999, 455999,
        456999, 457999, 458999, 459999, 460999, 461999, 462999, 463999,
        464999, 465999, 466999, 467999, 468999, 469999, 470999, 471999,
        472999, 473999, 474999, 475999, 476999, 477999, 478999, 479999,
        480999, 481999, 482999, 483999, 484999, 485999, 486999, 487999,
        488999, 489999, 490999, 491999, 492999, 493999, 494999, 495999,
        496999, 497999, 498999, 499999, 500999, 501999, 502999, 503999,
        504999, 505999, 506999, 507999, 508999, 509999, 510999, 511999,
        512999, 513999, 514999, 515999, 516999, 517999, 518999, 519999,
        520999, 521999, 522999, 523999, 524999, 525999, 526999, 527999,
        528999, 529999, 530999, 531999, 532999, 533999, 534999, 535999,
        536999, 537999, 538999, 539999, 540999, 541999, 542999, 543999,
        544999, 545999, 546999, 547999, 548999, 549999, 550999, 551999,
        552999, 553999, 554999, 555999, 556999, 557999, 558999, 559999,
        560999, 561999, 562999, 563999, 564999, 565999, 566999, 567999,
        568999, 569999, 570999, 571999, 572999, 573999, 574999, 575999,
        576999, 577999, 578999, 579999, 580999, 581999, 582999, 583999,
        584999, 585999, 586999, 587999, 588999, 589999, 590999, 591999,
        592999, 593999, 594999, 595999, 596999, 597999, 598999, 599999,
        600999, 601999, 602999, 603999, 604999, 605999, 606999, 607999,
        608999, 609999, 610999, 611999, 612999, 613999, 614999, 615999,
        616999, 617999, 618999, 619999, 620999, 621999, 622999, 623999,
        624999, 625999, 626999, 627999, 628999, 629999, 630999, 631999,
        632999, 633999, 634999, 635999, 636999, 637999, 638999, 639999,
        640999, 641999, 642999, 643999, 644999, 645999, 646999, 647999,
        648999, 649999, 650999, 651999, 652999, 653999, 654999, 655999,
        656999, 657999, 658999, 659999, 660999, 661999, 662999, 663999,
        664999, 665999, 666999, 667999, 668999, 669999, 670999, 671999,
        672999, 673999, 674999, 675999, 676999, 677999, 678999, 679999,
        680999, 681999, 682999, 683999, 684999, 685999, 686999, 687999,
        688999, 689999, 690999, 691999, 692999, 693999, 694999, 695999,
        696999, 697999, 698999, 699999, 700999, 701999, 702999, 703999,
        704999, 705999, 706999, 707999, 708999, 709999, 710999, 711999,
        712999, 713999, 714999, 715999, 716999, 717999, 718999, 719999,
        720999, 721999, 722999, 723999, 724999, 725999, 726999, 727999,
        728999, 729999, 730999, 731999, 732999, 733999, 734999, 735999,
        736999, 737999, 738999, 739999, 740999, 741999, 742999, 743999,
        744999, 745999, 746999, 747999, 748999, 749999, 750999, 751999,
        752999, 753999, 754999, 755999, 756999, 757999, 758999, 759999,
        760999, 761999, 762999, 763999, 764999, 765999, 766999, 767999,
        768999, 769999, 770999, 771999, 772999, 773999, 774999, 775999,
        776999, 777999, 778999, 779999, 780999, 781999, 782999, 783999,
        784999, 785999, 786999, 787999, 788999, 789999, 790999, 791999,
        792999, 793999, 794999, 795999, 796999, 797999, 798999, 799999,
        800999, 801999, 802999, 803999, 804999, 805999, 806999, 807999,
        808999, 809999, 810999, 811999, 812999, 813999, 814999, 815999,
        816999, 817999, 818999, 819999, 820999, 821999, 822999, 823999,
        824999, 825999, 826999, 827999, 828999, 829999, 830999, 831999,
        832999, 833999, 834999, 835999, 836999, 837999, 838999, 839999,
        840999, 841999, 842999, 843999, 844999, 845999, 846999, 847999,
        848999, 849999, 850999, 851999, 852999, 853999, 854999, 855999,
        856999, 857999, 858999, 859999, 860999, 861999, 862999, 863999,
        864999, 865999, 866999, 867999, 868999, 869999, 870999, 871999,
        872999, 873999, 874999, 875999, 876999, 877999, 878999, 879999,
        880999, 881999, 882999, 883999, 884999, 885999, 886999, 887999,
        888999, 889999, 890999, 891999, 892999, 893999, 894999, 895999,
        896999, 897999, 898999, 899999, 900999, 901999, 902999, 903999,
        904999, 905999, 906999, 907999, 908999, 909999, 910999, 911999,
        912999, 913999, 914999, 915999, 916999, 917999, 918999, 919999,
        920999, 921999, 922999, 923999, 924999, 925999, 926999, 927999,
        928999, 929999, 930999, 931999, 932999, 933999, 934999, 935999,
        936999, 937999, 938999, 939999, 940999, 941999, 942999, 943999,
        944999, 945999, 946999, 947999, 948999, 949999, 950999, 951999,
        952999, 953999, 954999, 955999, 956999, 957999, 958999, 959999,
        960999, 961999, 962999, 963999, 964999, 965999, 966999, 967999,
        968999, 969999, 970999, 971999, 972999, 973999, 974999, 975999,
        976999, 977999, 978999, 979999, 980999, 981999, 982999, 983999,
        984999, 985999, 986999, 987999, 988999, 989999, 990999, 991999,
        992999, 993999, 994999, 995999, 996999, 997999, 998999, 999999]),)

Graph View