import{s as $s,o as gs,n as R}from"../chunks/scheduler.defa9a21.js";import{S as Es,i as Cs,g as b,s as i,r as d,A as Ws,h as f,f as n,c as w,j as Xs,u as m,x as I,k as Rs,y as Vs,a as p,v as y,d as M,t as J,w as T}from"../chunks/index.fe795e71.js";import{T as _s}from"../chunks/Tip.179eb360.js";import{C as X}from"../chunks/CodeBlock.204b6c34.js";import{H as D,E as Ys}from"../chunks/getInferenceSnippets.2234a8dd.js";import{H as Ts,a as x}from"../chunks/HfOption.3c290b0f.js";function vs(h){let a,o;return a=new X({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdG9yY2gubm4ucGFyYWxsZWwlMjBpbXBvcnQlMjBEaXN0cmlidXRlZERhdGFQYXJhbGxlbCUyMGFzJTIwRERQJTBBZnJvbSUyMHRvcmNoLmRpc3RyaWJ1dGVkLmFsZ29yaXRobXMuZGRwX2NvbW1faG9va3MlMjBpbXBvcnQlMjBkZWZhdWx0X2hvb2tzJTBBZnJvbSUyMGFjY2VsZXJhdGUudGVzdF91dGlscy50ZXN0aW5nJTIwaW1wb3J0JTIwZ2V0X2JhY2tlbmQlMEElMEFkZXZpY2VfdHlwZSUyQyUyMF8lMkMlMjBfJTIwJTNEJTIwZ2V0X2JhY2tlbmQoKSUwQWRldmljZV9pZCUyMCUzRCUyMGdldGF0dHIodG9yY2glMkMlMjBkZXZpY2VfdHlwZSUyQyUyMHRvcmNoLmN1ZGEpLmN1cnJlbnRfZGV2aWNlKCklMEElMEFjbGFzcyUyME15TW9kZWwodG9yY2gubm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VwZXIoKS5fX2luaXRfXygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5sYXllciUyMCUzRCUyMHRvcmNoLm5uLkxpbmVhcigxMCUyQyUyMDEwKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMHgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwc2VsZi5sYXllcih4KSUwQSUwQW1vZGVsJTIwJTNEJTIwTXlNb2RlbCgpJTBBbW9kZWwlMjAlM0QlMjBERFAobW9kZWwlMkMlMjBkZXZpY2VfaWRzJTNEJTVCZGV2aWNlX2lkJTVEKSUwQW1vZGVsLnJlZ2lzdGVyX2NvbW1faG9vayhzdGF0ZSUzRE5vbmUlMkMlMjBob29rJTNEZGVmYXVsdF9ob29rcy5mcDE2X2NvbXByZXNzX2hvb2spJTBBJTBBJTIzJTIwVHJhaW5pbmclMjBsb29wJTBBZm9yJTIwZGF0YSUyQyUyMHRhcmdldHMlMjBpbiUyMGRhdGFfbG9hZGVyJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGRhdGEpJTBBJTIwJTIwJTIwJTIwbG9zcyUyMCUzRCUyMGNyaXRlcmlvbihvdXRwdXRzJTJDJTIwdGFyZ2V0cyklMEElMjAlMjAlMjAlMjBsb3NzLmJhY2t3YXJkKCklMEElMjAlMjAlMjAlMjBvcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyLnplcm9fZ3JhZCgp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> torch.nn.parallel <span class="hljs-keyword">import</span> DistributedDataParallel <span class="hljs-keyword">as</span> DDP
<span class="hljs-keyword">from</span> torch.distributed.algorithms.ddp_comm_hooks <span class="hljs-keyword">import</span> default_hooks
<span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend

device_type, _, _ = get_backend()
device_id = <span class="hljs-built_in">getattr</span>(torch, device_type, torch.cuda).current_device()

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModel</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.layer = torch.nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
        <span class="hljs-keyword">return</span> self.layer(x)

model = MyModel()
model = DDP(model, device_ids=[device_id])
model.register_comm_hook(state=<span class="hljs-literal">None</span>, hook=default_hooks.fp16_compress_hook)

<span class="hljs-comment"># Training loop</span>
<span class="hljs-keyword">for</span> data, targets <span class="hljs-keyword">in</span> data_loader:
    outputs = model(data)
    loss = criterion(outputs, targets)
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()`,wrap:!1}}),{c(){d(a.$$.fragment)},l(l){m(a.$$.fragment,l)},m(l,r){y(a,l,r),o=!0},p:R,i(l){o||(M(a.$$.fragment,l),o=!0)},o(l){J(a.$$.fragment,l),o=!1},d(l){T(a,l)}}}function Fs(h){let a,o;return a=new X({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUyQyUyMEREUENvbW11bmljYXRpb25Ib29rVHlwZSUyQyUyMERpc3RyaWJ1dGVkRGF0YVBhcmFsbGVsS3dhcmdzJTBBaW1wb3J0JTIwdG9yY2glMEElMEFjbGFzcyUyME15TW9kZWwodG9yY2gubm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VwZXIoKS5fX2luaXRfXygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5sYXllciUyMCUzRCUyMHRvcmNoLm5uLkxpbmVhcigxMCUyQyUyMDEwKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMHgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwc2VsZi5sYXllcih4KSUwQSUwQSUyMyUyMEREUCUyMENvbW11bmljYXRpb24lMjBIb29rJTIwc2V0dXAlMEFkZHBfa3dhcmdzJTIwJTNEJTIwRGlzdHJpYnV0ZWREYXRhUGFyYWxsZWxLd2FyZ3MoY29tbV9ob29rJTNERERQQ29tbXVuaWNhdGlvbkhvb2tUeXBlLkZQMTYpJTBBYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcihrd2FyZ3NfaGFuZGxlcnMlM0QlNUJkZHBfa3dhcmdzJTVEKSUwQSUwQW1vZGVsJTIwJTNEJTIwTXlNb2RlbCgpJTBBb3B0aW1pemVyJTIwJTNEJTIwdG9yY2gub3B0aW0uQWRhbShtb2RlbC5wYXJhbWV0ZXJzKCkpJTBBZGF0YV9sb2FkZXIlMjAlM0QlMjBEYXRhTG9hZGVyKGRhdGFzZXQlMkMlMjBiYXRjaF9zaXplJTNEMTYpJTBBJTBBbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjBkYXRhX2xvYWRlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUobW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjBkYXRhX2xvYWRlciklMEElMEElMjMlMjBUcmFpbmluZyUyMGxvb3AlMEFmb3IlMjBkYXRhJTJDJTIwdGFyZ2V0cyUyMGluJTIwZGF0YV9sb2FkZXIlM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoZGF0YSklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwY3JpdGVyaW9uKG91dHB1dHMlMkMlMjB0YXJnZXRzKSUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMG9wdGltaXplci56ZXJvX2dyYWQoKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator, DDPCommunicationHookType, DistributedDataParallelKwargs
<span class="hljs-keyword">import</span> torch

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModel</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.layer = torch.nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
        <span class="hljs-keyword">return</span> self.layer(x)

<span class="hljs-comment"># DDP Communication Hook setup</span>
ddp_kwargs = DistributedDataParallelKwargs(comm_hook=DDPCommunicationHookType.FP16)
accelerator = Accelerator(kwargs_handlers=[ddp_kwargs])

model = MyModel()
optimizer = torch.optim.Adam(model.parameters())
data_loader = DataLoader(dataset, batch_size=<span class="hljs-number">16</span>)

model, optimizer, data_loader = accelerator.prepare(model, optimizer, data_loader)

<span class="hljs-comment"># Training loop</span>
<span class="hljs-keyword">for</span> data, targets <span class="hljs-keyword">in</span> data_loader:
    outputs = model(data)
    loss = criterion(outputs, targets)
    accelerator.backward(loss)
    optimizer.step()
    optimizer.zero_grad()`,wrap:!1}}),{c(){d(a.$$.fragment)},l(l){m(a.$$.fragment,l)},m(l,r){y(a,l,r),o=!0},p:R,i(l){o||(M(a.$$.fragment,l),o=!0)},o(l){J(a.$$.fragment,l),o=!1},d(l){T(a,l)}}}function Qs(h){let a,o,l,r;return a=new x({props:{id:"fp16",option:"PyTorch",$$slots:{default:[vs]},$$scope:{ctx:h}}}),l=new x({props:{id:"fp16",option:"Accelerate",$$slots:{default:[Fs]},$$scope:{ctx:h}}}),{c(){d(a.$$.fragment),o=i(),d(l.$$.fragment)},l(e){m(a.$$.fragment,e),o=w(e),m(l.$$.fragment,e)},m(e,c){y(a,e,c),p(e,o,c),y(l,e,c),r=!0},p(e,c){const U={};c&2&&(U.$$scope={dirty:c,ctx:e}),a.$set(U);const u={};c&2&&(u.$$scope={dirty:c,ctx:e}),l.$set(u)},i(e){r||(M(a.$$.fragment,e),M(l.$$.fragment,e),r=!0)},o(e){J(a.$$.fragment,e),J(l.$$.fragment,e),r=!1},d(e){e&&n(o),T(a,e),T(l,e)}}}function Ns(h){let a,o="BF16 Compression Hook API is experimental, and it requires NCCL version later than 2.9.6.";return{c(){a=b("p"),a.textContent=o},l(l){a=f(l,"P",{"data-svelte-h":!0}),I(a)!=="svelte-1s0p8uc"&&(a.textContent=o)},m(l,r){p(l,a,r)},p:R,d(l){l&&n(a)}}}function Ss(h){let a,o;return a=new X({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdG9yY2gubm4ucGFyYWxsZWwlMjBpbXBvcnQlMjBEaXN0cmlidXRlZERhdGFQYXJhbGxlbCUyMGFzJTIwRERQJTBBZnJvbSUyMHRvcmNoLmRpc3RyaWJ1dGVkLmFsZ29yaXRobXMuZGRwX2NvbW1faG9va3MlMjBpbXBvcnQlMjBkZWZhdWx0X2hvb2tzJTBBZnJvbSUyMGFjY2VsZXJhdGUudGVzdF91dGlscy50ZXN0aW5nJTIwaW1wb3J0JTIwZ2V0X2JhY2tlbmQlMEElMEFkZXZpY2VfdHlwZSUyQyUyMF8lMkMlMjBfJTIwJTNEJTIwZ2V0X2JhY2tlbmQoKSUwQWRldmljZV9pZCUyMCUzRCUyMGdldGF0dHIodG9yY2glMkMlMjBkZXZpY2VfdHlwZSUyQyUyMHRvcmNoLmN1ZGEpLmN1cnJlbnRfZGV2aWNlKCklMEElMEFjbGFzcyUyME15TW9kZWwodG9yY2gubm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VwZXIoKS5fX2luaXRfXygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5sYXllciUyMCUzRCUyMHRvcmNoLm5uLkxpbmVhcigxMCUyQyUyMDEwKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMHgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwc2VsZi5sYXllcih4KSUwQSUwQW1vZGVsJTIwJTNEJTIwTXlNb2RlbCgpJTBBbW9kZWwlMjAlM0QlMjBERFAobW9kZWwlMkMlMjBkZXZpY2VfaWRzJTNEJTVCZGV2aWNlX2lkJTVEKSUwQW1vZGVsLnJlZ2lzdGVyX2NvbW1faG9vayhzdGF0ZSUzRE5vbmUlMkMlMjBob29rJTNEZGVmYXVsdF9ob29rcy5iZjE2X2NvbXByZXNzX2hvb2spJTBBJTBBJTIzJTIwVHJhaW5pbmclMjBsb29wJTBBZm9yJTIwZGF0YSUyQyUyMHRhcmdldHMlMjBpbiUyMGRhdGFfbG9hZGVyJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGRhdGEpJTBBJTIwJTIwJTIwJTIwbG9zcyUyMCUzRCUyMGNyaXRlcmlvbihvdXRwdXRzJTJDJTIwdGFyZ2V0cyklMEElMjAlMjAlMjAlMjBsb3NzLmJhY2t3YXJkKCklMEElMjAlMjAlMjAlMjBvcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyLnplcm9fZ3JhZCgp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> torch.nn.parallel <span class="hljs-keyword">import</span> DistributedDataParallel <span class="hljs-keyword">as</span> DDP
<span class="hljs-keyword">from</span> torch.distributed.algorithms.ddp_comm_hooks <span class="hljs-keyword">import</span> default_hooks
<span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend

device_type, _, _ = get_backend()
device_id = <span class="hljs-built_in">getattr</span>(torch, device_type, torch.cuda).current_device()

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModel</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.layer = torch.nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
        <span class="hljs-keyword">return</span> self.layer(x)

model = MyModel()
model = DDP(model, device_ids=[device_id])
model.register_comm_hook(state=<span class="hljs-literal">None</span>, hook=default_hooks.bf16_compress_hook)

<span class="hljs-comment"># Training loop</span>
<span class="hljs-keyword">for</span> data, targets <span class="hljs-keyword">in</span> data_loader:
    outputs = model(data)
    loss = criterion(outputs, targets)
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()`,wrap:!1}}),{c(){d(a.$$.fragment)},l(l){m(a.$$.fragment,l)},m(l,r){y(a,l,r),o=!0},p:R,i(l){o||(M(a.$$.fragment,l),o=!0)},o(l){J(a.$$.fragment,l),o=!1},d(l){T(a,l)}}}function zs(h){let a,o;return a=new X({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUyQyUyMEREUENvbW11bmljYXRpb25Ib29rVHlwZSUyQyUyMERpc3RyaWJ1dGVkRGF0YVBhcmFsbGVsS3dhcmdzJTBBaW1wb3J0JTIwdG9yY2glMEElMEFjbGFzcyUyME15TW9kZWwodG9yY2gubm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VwZXIoKS5fX2luaXRfXygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5sYXllciUyMCUzRCUyMHRvcmNoLm5uLkxpbmVhcigxMCUyQyUyMDEwKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMHgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwc2VsZi5sYXllcih4KSUwQSUwQSUyMyUyMEREUCUyMENvbW11bmljYXRpb24lMjBIb29rJTIwc2V0dXAlMEFkZHBfa3dhcmdzJTIwJTNEJTIwRGlzdHJpYnV0ZWREYXRhUGFyYWxsZWxLd2FyZ3MoY29tbV9ob29rJTNERERQQ29tbXVuaWNhdGlvbkhvb2tUeXBlLkJGMTYpJTBBYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcihrd2FyZ3NfaGFuZGxlcnMlM0QlNUJkZHBfa3dhcmdzJTVEKSUwQSUwQW1vZGVsJTIwJTNEJTIwTXlNb2RlbCgpJTBBb3B0aW1pemVyJTIwJTNEJTIwdG9yY2gub3B0aW0uQWRhbShtb2RlbC5wYXJhbWV0ZXJzKCkpJTBBZGF0YV9sb2FkZXIlMjAlM0QlMjBEYXRhTG9hZGVyKGRhdGFzZXQlMkMlMjBiYXRjaF9zaXplJTNEMTYpJTBBJTBBbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjBkYXRhX2xvYWRlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUobW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjBkYXRhX2xvYWRlciklMEElMEElMjMlMjBUcmFpbmluZyUyMGxvb3AlMEFmb3IlMjBkYXRhJTJDJTIwdGFyZ2V0cyUyMGluJTIwZGF0YV9sb2FkZXIlM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoZGF0YSklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwY3JpdGVyaW9uKG91dHB1dHMlMkMlMjB0YXJnZXRzKSUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMG9wdGltaXplci56ZXJvX2dyYWQoKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator, DDPCommunicationHookType, DistributedDataParallelKwargs
<span class="hljs-keyword">import</span> torch

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModel</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.layer = torch.nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
        <span class="hljs-keyword">return</span> self.layer(x)

<span class="hljs-comment"># DDP Communication Hook setup</span>
ddp_kwargs = DistributedDataParallelKwargs(comm_hook=DDPCommunicationHookType.BF16)
accelerator = Accelerator(kwargs_handlers=[ddp_kwargs])

model = MyModel()
optimizer = torch.optim.Adam(model.parameters())
data_loader = DataLoader(dataset, batch_size=<span class="hljs-number">16</span>)

model, optimizer, data_loader = accelerator.prepare(model, optimizer, data_loader)

<span class="hljs-comment"># Training loop</span>
<span class="hljs-keyword">for</span> data, targets <span class="hljs-keyword">in</span> data_loader:
    outputs = model(data)
    loss = criterion(outputs, targets)
    accelerator.backward(loss)
    optimizer.step()
    optimizer.zero_grad()`,wrap:!1}}),{c(){d(a.$$.fragment)},l(l){m(a.$$.fragment,l)},m(l,r){y(a,l,r),o=!0},p:R,i(l){o||(M(a.$$.fragment,l),o=!0)},o(l){J(a.$$.fragment,l),o=!1},d(l){T(a,l)}}}function Hs(h){let a,o,l,r;return a=new x({props:{id:"bf16",option:"PyTorch",$$slots:{default:[Ss]},$$scope:{ctx:h}}}),l=new x({props:{id:"bf16",option:"Accelerate",$$slots:{default:[zs]},$$scope:{ctx:h}}}),{c(){d(a.$$.fragment),o=i(),d(l.$$.fragment)},l(e){m(a.$$.fragment,e),o=w(e),m(l.$$.fragment,e)},m(e,c){y(a,e,c),p(e,o,c),y(l,e,c),r=!0},p(e,c){const U={};c&2&&(U.$$scope={dirty:c,ctx:e}),a.$set(U);const u={};c&2&&(u.$$scope={dirty:c,ctx:e}),l.$set(u)},i(e){r||(M(a.$$.fragment,e),M(l.$$.fragment,e),r=!0)},o(e){J(a.$$.fragment,e),J(l.$$.fragment,e),r=!1},d(e){e&&n(o),T(a,e),T(l,e)}}}function Ds(h){let a,o="PowerSGD typically requires extra memory of the same size as the model’s gradients to enable error feedback, which can compensate for biased compressed communication and improve accuracy.";return{c(){a=b("p"),a.textContent=o},l(l){a=f(l,"P",{"data-svelte-h":!0}),I(a)!=="svelte-1f2etf0"&&(a.textContent=o)},m(l,r){p(l,a,r)},p:R,d(l){l&&n(a)}}}function xs(h){let a,o;return a=new X({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdG9yY2gubm4ucGFyYWxsZWwlMjBpbXBvcnQlMjBEaXN0cmlidXRlZERhdGFQYXJhbGxlbCUyMGFzJTIwRERQJTBBZnJvbSUyMHRvcmNoLmRpc3RyaWJ1dGVkLmFsZ29yaXRobXMuZGRwX2NvbW1faG9va3MlMjBpbXBvcnQlMjBwb3dlclNHRF9ob29rJTBBZnJvbSUyMGFjY2VsZXJhdGUudGVzdF91dGlscy50ZXN0aW5nJTIwaW1wb3J0JTIwZ2V0X2JhY2tlbmQlMEElMEFkZXZpY2VfdHlwZSUyQyUyMF8lMkMlMjBfJTIwJTNEJTIwZ2V0X2JhY2tlbmQoKSUwQWRldmljZV9pZCUyMCUzRCUyMGdldGF0dHIodG9yY2glMkMlMjBkZXZpY2VfdHlwZSUyQyUyMHRvcmNoLmN1ZGEpLmN1cnJlbnRfZGV2aWNlKCklMEElMEFjbGFzcyUyME15TW9kZWwodG9yY2gubm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VwZXIoKS5fX2luaXRfXygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5sYXllciUyMCUzRCUyMHRvcmNoLm5uLkxpbmVhcigxMCUyQyUyMDEwKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMHgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwc2VsZi5sYXllcih4KSUwQSUwQW1vZGVsJTIwJTNEJTIwTXlNb2RlbCgpJTBBbW9kZWwlMjAlM0QlMjBERFAobW9kZWwlMkMlMjBkZXZpY2VfaWRzJTNEJTVCZGV2aWNlX2lkJTVEKSUwQXN0YXRlJTIwJTNEJTIwcG93ZXJTR0RfaG9vay5Qb3dlclNHRFN0YXRlKHByb2Nlc3NfZ3JvdXAlM0ROb25lKSUwQW1vZGVsLnJlZ2lzdGVyX2NvbW1faG9vayhzdGF0ZSUzRHN0YXRlJTJDJTIwaG9vayUzRHBvd2VyU0dEX2hvb2sucG93ZXJTR0RfaG9vayklMEElMEElMjMlMjBUcmFpbmluZyUyMGxvb3AlMEFmb3IlMjBkYXRhJTJDJTIwdGFyZ2V0cyUyMGluJTIwZGF0YV9sb2FkZXIlM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoZGF0YSklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwY3JpdGVyaW9uKG91dHB1dHMlMkMlMjB0YXJnZXRzKSUwQSUyMCUyMCUyMCUyMGxvc3MuYmFja3dhcmQoKSUwQSUyMCUyMCUyMCUyMG9wdGltaXplci5zdGVwKCklMEElMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> torch.nn.parallel <span class="hljs-keyword">import</span> DistributedDataParallel <span class="hljs-keyword">as</span> DDP
<span class="hljs-keyword">from</span> torch.distributed.algorithms.ddp_comm_hooks <span class="hljs-keyword">import</span> powerSGD_hook
<span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend

device_type, _, _ = get_backend()
device_id = <span class="hljs-built_in">getattr</span>(torch, device_type, torch.cuda).current_device()

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModel</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.layer = torch.nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
        <span class="hljs-keyword">return</span> self.layer(x)

model = MyModel()
model = DDP(model, device_ids=[device_id])
state = powerSGD_hook.PowerSGDState(process_group=<span class="hljs-literal">None</span>)
model.register_comm_hook(state=state, hook=powerSGD_hook.powerSGD_hook)

<span class="hljs-comment"># Training loop</span>
<span class="hljs-keyword">for</span> data, targets <span class="hljs-keyword">in</span> data_loader:
    outputs = model(data)
    loss = criterion(outputs, targets)
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()`,wrap:!1}}),{c(){d(a.$$.fragment)},l(l){m(a.$$.fragment,l)},m(l,r){y(a,l,r),o=!0},p:R,i(l){o||(M(a.$$.fragment,l),o=!0)},o(l){J(a.$$.fragment,l),o=!1},d(l){T(a,l)}}}function Ks(h){let a,o;return a=new X({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUyQyUyMEREUENvbW11bmljYXRpb25Ib29rVHlwZSUyQyUyMERpc3RyaWJ1dGVkRGF0YVBhcmFsbGVsS3dhcmdzJTBBaW1wb3J0JTIwdG9yY2glMEElMEFjbGFzcyUyME15TW9kZWwodG9yY2gubm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VwZXIoKS5fX2luaXRfXygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5sYXllciUyMCUzRCUyMHRvcmNoLm5uLkxpbmVhcigxMCUyQyUyMDEwKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMHgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwc2VsZi5sYXllcih4KSUwQSUwQSUyMyUyMEREUCUyMENvbW11bmljYXRpb24lMjBIb29rJTIwc2V0dXAlMEFkZHBfa3dhcmdzJTIwJTNEJTIwRGlzdHJpYnV0ZWREYXRhUGFyYWxsZWxLd2FyZ3MoY29tbV9ob29rJTNERERQQ29tbXVuaWNhdGlvbkhvb2tUeXBlLlBPV0VSX1NHRCklMEFhY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKGt3YXJnc19oYW5kbGVycyUzRCU1QmRkcF9rd2FyZ3MlNUQpJTBBJTBBbW9kZWwlMjAlM0QlMjBNeU1vZGVsKCklMEFvcHRpbWl6ZXIlMjAlM0QlMjB0b3JjaC5vcHRpbS5BZGFtKG1vZGVsLnBhcmFtZXRlcnMoKSklMEFkYXRhX2xvYWRlciUyMCUzRCUyMERhdGFMb2FkZXIoZGF0YXNldCUyQyUyMGJhdGNoX3NpemUlM0QxNiklMEElMEFtb2RlbCUyQyUyMG9wdGltaXplciUyQyUyMGRhdGFfbG9hZGVyJTIwJTNEJTIwYWNjZWxlcmF0b3IucHJlcGFyZShtb2RlbCUyQyUyMG9wdGltaXplciUyQyUyMGRhdGFfbG9hZGVyKSUwQSUwQSUyMyUyMFRyYWluaW5nJTIwbG9vcCUwQWZvciUyMGRhdGElMkMlMjB0YXJnZXRzJTIwaW4lMjBkYXRhX2xvYWRlciUzQSUwQSUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbChkYXRhKSUwQSUyMCUyMCUyMCUyMGxvc3MlMjAlM0QlMjBjcml0ZXJpb24ob3V0cHV0cyUyQyUyMHRhcmdldHMpJTBBJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IuYmFja3dhcmQobG9zcyklMEElMjAlMjAlMjAlMjBvcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyLnplcm9fZ3JhZCgp",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator, DDPCommunicationHookType, DistributedDataParallelKwargs
<span class="hljs-keyword">import</span> torch

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModel</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.layer = torch.nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
        <span class="hljs-keyword">return</span> self.layer(x)

<span class="hljs-comment"># DDP Communication Hook setup</span>
ddp_kwargs = DistributedDataParallelKwargs(comm_hook=DDPCommunicationHookType.POWER_SGD)
accelerator = Accelerator(kwargs_handlers=[ddp_kwargs])

model = MyModel()
optimizer = torch.optim.Adam(model.parameters())
data_loader = DataLoader(dataset, batch_size=<span class="hljs-number">16</span>)

model, optimizer, data_loader = accelerator.prepare(model, optimizer, data_loader)

<span class="hljs-comment"># Training loop</span>
<span class="hljs-keyword">for</span> data, targets <span class="hljs-keyword">in</span> data_loader:
    outputs = model(data)
    loss = criterion(outputs, targets)
    accelerator.backward(loss)
    optimizer.step()
    optimizer.zero_grad()`,wrap:!1}}),{c(){d(a.$$.fragment)},l(l){m(a.$$.fragment,l)},m(l,r){y(a,l,r),o=!0},p:R,i(l){o||(M(a.$$.fragment,l),o=!0)},o(l){J(a.$$.fragment,l),o=!1},d(l){T(a,l)}}}function As(h){let a,o,l,r;return a=new x({props:{id:"powerSGD",option:"PyTorch",$$slots:{default:[xs]},$$scope:{ctx:h}}}),l=new x({props:{id:"powerSGD",option:"Accelerate",$$slots:{default:[Ks]},$$scope:{ctx:h}}}),{c(){d(a.$$.fragment),o=i(),d(l.$$.fragment)},l(e){m(a.$$.fragment,e),o=w(e),m(l.$$.fragment,e)},m(e,c){y(a,e,c),p(e,o,c),y(l,e,c),r=!0},p(e,c){const U={};c&2&&(U.$$scope={dirty:c,ctx:e}),a.$set(U);const u={};c&2&&(u.$$scope={dirty:c,ctx:e}),l.$set(u)},i(e){r||(M(a.$$.fragment,e),M(l.$$.fragment,e),r=!0)},o(e){J(a.$$.fragment,e),J(l.$$.fragment,e),r=!1},d(e){e&&n(o),T(a,e),T(l,e)}}}function Ps(h){let a,o,l,r,e,c,U,u="Distributed Data Parallel (DDP) communication hooks provide a generic interface to control how gradients are communicated across workers by overriding the vanilla allreduce in <code>DistributedDataParallel</code>. A few built-in communication hooks are provided, and users can easily apply any of these hooks to optimize communication.",A,_,hs="<li><strong>FP16 Compression Hook</strong>: Compresses gradients by casting them to half-precision floating-point format (<code>torch.float16</code>), reducing communication overhead.</li> <li><strong>BF16 Compression Hook</strong>: Similar to FP16, but uses the Brain Floating Point format (<code>torch.bfloat16</code>), which can be more efficient on certain hardware.</li> <li><strong>PowerSGD Hook</strong>: An advanced gradient compression algorithm that provides high compression rates and can accelerate bandwidth-bound distributed training.</li>",P,$,Us="In this tutorial, you will see how to quickly set up DDP communication hooks and perform training with the utilities provided in Accelerate, which can be as simple as adding just one new line of code! This demonstrates how to use DDP communication hooks to optimize gradient communication in distributed training with the Accelerate library.",L,g,q,j,O,E,ss,k,ls,G,as,C,es,B,ts,Z,os,W,ns,V,bs="There are two additional utilities for supporting optional functionalities with the communication hooks.",ps,Y,rs,v,fs="<code>comm_wrapper</code> is an option to wrap a communication hook with additional functionality. For example, it can be used to combine FP16 compression with other communication strategies. Currently supported wrappers are <code>no</code>, <code>fp16</code>, and <code>bf16</code>.",cs,F,is,Q,ws,N,us="<code>comm_state_option</code> allows you to pass additional state information required by certain communication hooks. This is particularly useful for stateful hooks like <code>PowerSGD</code>, which require maintaining hyperparameters and internal states across training steps. Below is an example showcasing the use of <code>comm_state_option</code> with the <code>PowerSGD</code> hook.",ds,S,ms,z,Is='For more advanced usage and additional hooks, refer to the <a href="https://pytorch.org/docs/stable/ddp_comm_hooks.html" rel="nofollow">PyTorch DDP Communication Hooks documentation</a>.',ys,H,Ms,K,Js;return e=new D({props:{title:"DDP Communication Hooks",local:"ddp-communication-hooks",headingTag:"h1"}}),g=new D({props:{title:"FP16 Compression Hook",local:"fp16-compression-hook",headingTag:"h2"}}),j=new Ts({props:{id:"fp16",options:["PyTorch","Accelerate"],$$slots:{default:[Qs]},$$scope:{ctx:h}}}),E=new D({props:{title:"BF16 Compression Hook",local:"bf16-compression-hook",headingTag:"h3"}}),k=new _s({props:{warning:!0,$$slots:{default:[Ns]},$$scope:{ctx:h}}}),G=new Ts({props:{id:"bf16",options:["PyTorch","Accelerate"],$$slots:{default:[Hs]},$$scope:{ctx:h}}}),C=new D({props:{title:"PowerSGD Hook",local:"powersgd-hook",headingTag:"h3"}}),B=new _s({props:{warning:!0,$$slots:{default:[Ds]},$$scope:{ctx:h}}}),Z=new Ts({props:{id:"powerSGD",options:["PyTorch","Accelerate"],$$slots:{default:[As]},$$scope:{ctx:h}}}),W=new D({props:{title:"DDP Communication Hooks utilities",local:"ddp-communication-hooks-utilities",headingTag:"h2"}}),Y=new D({props:{title:"comm_wrapper",local:"commwrapper",headingTag:"h3"}}),F=new X({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUyQyUyMEREUENvbW11bmljYXRpb25Ib29rVHlwZSUyQyUyMERpc3RyaWJ1dGVkRGF0YVBhcmFsbGVsS3dhcmdzJTBBaW1wb3J0JTIwdG9yY2glMEElMEFjbGFzcyUyME15TW9kZWwodG9yY2gubm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VwZXIoKS5fX2luaXRfXygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5sYXllciUyMCUzRCUyMHRvcmNoLm5uLkxpbmVhcigxMCUyQyUyMDEwKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMHgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwc2VsZi5sYXllcih4KSUwQSUwQSUyMyUyMEREUCUyMENvbW11bmljYXRpb24lMjBIb29rJTIwc2V0dXAlMEFkZHBfa3dhcmdzJTIwJTNEJTIwRGlzdHJpYnV0ZWREYXRhUGFyYWxsZWxLd2FyZ3MoJTBBJTIwJTIwJTIwJTIwY29tbV9ob29rJTNERERQQ29tbXVuaWNhdGlvbkhvb2tUeXBlLlBPV0VSX1NHRCUyQyUwQSUyMCUyMCUyMCUyMGNvbW1fd3JhcHBlciUzREREUENvbW11bmljYXRpb25Ib29rVHlwZS5GUDE2JTBBKSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3Ioa3dhcmdzX2hhbmRsZXJzJTNEJTVCZGRwX2t3YXJncyU1RCklMEElMEFtb2RlbCUyMCUzRCUyME15TW9kZWwoKSUwQW9wdGltaXplciUyMCUzRCUyMHRvcmNoLm9wdGltLkFkYW0obW9kZWwucGFyYW1ldGVycygpKSUwQWRhdGFfbG9hZGVyJTIwJTNEJTIwRGF0YUxvYWRlcihkYXRhc2V0JTJDJTIwYmF0Y2hfc2l6ZSUzRDE2KSUwQSUwQW1vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwZGF0YV9sb2FkZXIlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKG1vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwZGF0YV9sb2FkZXIpJTBBJTBBJTIzJTIwVHJhaW5pbmclMjBsb29wJTBBZm9yJTIwZGF0YSUyQyUyMHRhcmdldHMlMjBpbiUyMGRhdGFfbG9hZGVyJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGRhdGEpJTBBJTIwJTIwJTIwJTIwbG9zcyUyMCUzRCUyMGNyaXRlcmlvbihvdXRwdXRzJTJDJTIwdGFyZ2V0cyklMEElMjAlMjAlMjAlMjBhY2NlbGVyYXRvci5iYWNrd2FyZChsb3NzKSUwQSUyMCUyMCUyMCUyMG9wdGltaXplci5zdGVwKCklMEElMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCk=",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator, DDPCommunicationHookType, DistributedDataParallelKwargs
<span class="hljs-keyword">import</span> torch

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModel</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.layer = torch.nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
        <span class="hljs-keyword">return</span> self.layer(x)

<span class="hljs-comment"># DDP Communication Hook setup</span>
ddp_kwargs = DistributedDataParallelKwargs(
    comm_hook=DDPCommunicationHookType.POWER_SGD,
    comm_wrapper=DDPCommunicationHookType.FP16
)
accelerator = Accelerator(kwargs_handlers=[ddp_kwargs])

model = MyModel()
optimizer = torch.optim.Adam(model.parameters())
data_loader = DataLoader(dataset, batch_size=<span class="hljs-number">16</span>)

model, optimizer, data_loader = accelerator.prepare(model, optimizer, data_loader)

<span class="hljs-comment"># Training loop</span>
<span class="hljs-keyword">for</span> data, targets <span class="hljs-keyword">in</span> data_loader:
    outputs = model(data)
    loss = criterion(outputs, targets)
    accelerator.backward(loss)
    optimizer.step()
    optimizer.zero_grad()`,wrap:!1}}),Q=new D({props:{title:"comm_state_option",local:"commstateoption",headingTag:"h3"}}),S=new X({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUyQyUyMEREUENvbW11bmljYXRpb25Ib29rVHlwZSUyQyUyMERpc3RyaWJ1dGVkRGF0YVBhcmFsbGVsS3dhcmdzJTBBaW1wb3J0JTIwdG9yY2glMEElMEFjbGFzcyUyME15TW9kZWwodG9yY2gubm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VwZXIoKS5fX2luaXRfXygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5sYXllciUyMCUzRCUyMHRvcmNoLm5uLkxpbmVhcigxMCUyQyUyMDEwKSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMHgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwc2VsZi5sYXllcih4KSUwQSUwQSUyMyUyMEREUCUyMENvbW11bmljYXRpb24lMjBIb29rJTIwc2V0dXAlMEFkZHBfa3dhcmdzJTIwJTNEJTIwRGlzdHJpYnV0ZWREYXRhUGFyYWxsZWxLd2FyZ3MoJTBBJTIwJTIwJTIwJTIwY29tbV9ob29rJTNERERQQ29tbXVuaWNhdGlvbkhvb2tUeXBlLlBPV0VSX1NHRCUyQyUwQSUyMCUyMCUyMCUyMGNvbW1fc3RhdGVfb3B0aW9uJTNEJTdCJTIybWF0cml4X2FwcHJveGltYXRpb25fcmFuayUyMiUzQSUyMDIlN0QlMEEpJTBBYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcihrd2FyZ3NfaGFuZGxlcnMlM0QlNUJkZHBfa3dhcmdzJTVEKSUwQSUwQW1vZGVsJTIwJTNEJTIwTXlNb2RlbCgpJTBBb3B0aW1pemVyJTIwJTNEJTIwdG9yY2gub3B0aW0uQWRhbShtb2RlbC5wYXJhbWV0ZXJzKCkpJTBBZGF0YV9sb2FkZXIlMjAlM0QlMjBEYXRhTG9hZGVyKGRhdGFzZXQlMkMlMjBiYXRjaF9zaXplJTNEMTYpJTBBJTBBbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjBkYXRhX2xvYWRlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUobW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjBkYXRhX2xvYWRlciklMEElMEElMjMlMjBUcmFpbmluZyUyMGxvb3AlMEFmb3IlMjBkYXRhJTJDJTIwdGFyZ2V0cyUyMGluJTIwZGF0YV9sb2FkZXIlM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoZGF0YSklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwY3JpdGVyaW9uKG91dHB1dHMlMkMlMjB0YXJnZXRzKSUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMG9wdGltaXplci56ZXJvX2dyYWQoKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator, DDPCommunicationHookType, DistributedDataParallelKwargs
<span class="hljs-keyword">import</span> torch

<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModel</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.layer = torch.nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):
        <span class="hljs-keyword">return</span> self.layer(x)

<span class="hljs-comment"># DDP Communication Hook setup</span>
ddp_kwargs = DistributedDataParallelKwargs(
    comm_hook=DDPCommunicationHookType.POWER_SGD,
    comm_state_option={<span class="hljs-string">&quot;matrix_approximation_rank&quot;</span>: <span class="hljs-number">2</span>}
)
accelerator = Accelerator(kwargs_handlers=[ddp_kwargs])

model = MyModel()
optimizer = torch.optim.Adam(model.parameters())
data_loader = DataLoader(dataset, batch_size=<span class="hljs-number">16</span>)

model, optimizer, data_loader = accelerator.prepare(model, optimizer, data_loader)

<span class="hljs-comment"># Training loop</span>
<span class="hljs-keyword">for</span> data, targets <span class="hljs-keyword">in</span> data_loader:
    outputs = model(data)
    loss = criterion(outputs, targets)
    accelerator.backward(loss)
    optimizer.step()
    optimizer.zero_grad()`,wrap:!1}}),H=new Ys({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/ddp_comm_hook.md"}}),{c(){a=b("meta"),o=i(),l=b("p"),r=i(),d(e.$$.fragment),c=i(),U=b("p"),U.innerHTML=u,A=i(),_=b("ul"),_.innerHTML=hs,P=i(),$=b("p"),$.textContent=Us,L=i(),d(g.$$.fragment),q=i(),d(j.$$.fragment),O=i(),d(E.$$.fragment),ss=i(),d(k.$$.fragment),ls=i(),d(G.$$.fragment),as=i(),d(C.$$.fragment),es=i(),d(B.$$.fragment),ts=i(),d(Z.$$.fragment),os=i(),d(W.$$.fragment),ns=i(),V=b("p"),V.textContent=bs,ps=i(),d(Y.$$.fragment),rs=i(),v=b("p"),v.innerHTML=fs,cs=i(),d(F.$$.fragment),is=i(),d(Q.$$.fragment),ws=i(),N=b("p"),N.innerHTML=us,ds=i(),d(S.$$.fragment),ms=i(),z=b("p"),z.innerHTML=Is,ys=i(),d(H.$$.fragment),Ms=i(),K=b("p"),this.h()},l(s){const t=Ws("svelte-u9bgzb",document.head);a=f(t,"META",{name:!0,content:!0}),t.forEach(n),o=w(s),l=f(s,"P",{}),Xs(l).forEach(n),r=w(s),m(e.$$.fragment,s),c=w(s),U=f(s,"P",{"data-svelte-h":!0}),I(U)!=="svelte-x6tyko"&&(U.innerHTML=u),A=w(s),_=f(s,"UL",{"data-svelte-h":!0}),I(_)!=="svelte-zxp3oz"&&(_.innerHTML=hs),P=w(s),$=f(s,"P",{"data-svelte-h":!0}),I($)!=="svelte-10ve3hd"&&($.textContent=Us),L=w(s),m(g.$$.fragment,s),q=w(s),m(j.$$.fragment,s),O=w(s),m(E.$$.fragment,s),ss=w(s),m(k.$$.fragment,s),ls=w(s),m(G.$$.fragment,s),as=w(s),m(C.$$.fragment,s),es=w(s),m(B.$$.fragment,s),ts=w(s),m(Z.$$.fragment,s),os=w(s),m(W.$$.fragment,s),ns=w(s),V=f(s,"P",{"data-svelte-h":!0}),I(V)!=="svelte-1nbipsx"&&(V.textContent=bs),ps=w(s),m(Y.$$.fragment,s),rs=w(s),v=f(s,"P",{"data-svelte-h":!0}),I(v)!=="svelte-oi3r9p"&&(v.innerHTML=fs),cs=w(s),m(F.$$.fragment,s),is=w(s),m(Q.$$.fragment,s),ws=w(s),N=f(s,"P",{"data-svelte-h":!0}),I(N)!=="svelte-1igm2b3"&&(N.innerHTML=us),ds=w(s),m(S.$$.fragment,s),ms=w(s),z=f(s,"P",{"data-svelte-h":!0}),I(z)!=="svelte-1ll6vn"&&(z.innerHTML=Is),ys=w(s),m(H.$$.fragment,s),Ms=w(s),K=f(s,"P",{}),Xs(K).forEach(n),this.h()},h(){Rs(a,"name","hf:doc:metadata"),Rs(a,"content",Ls)},m(s,t){Vs(document.head,a),p(s,o,t),p(s,l,t),p(s,r,t),y(e,s,t),p(s,c,t),p(s,U,t),p(s,A,t),p(s,_,t),p(s,P,t),p(s,$,t),p(s,L,t),y(g,s,t),p(s,q,t),y(j,s,t),p(s,O,t),y(E,s,t),p(s,ss,t),y(k,s,t),p(s,ls,t),y(G,s,t),p(s,as,t),y(C,s,t),p(s,es,t),y(B,s,t),p(s,ts,t),y(Z,s,t),p(s,os,t),y(W,s,t),p(s,ns,t),p(s,V,t),p(s,ps,t),y(Y,s,t),p(s,rs,t),p(s,v,t),p(s,cs,t),y(F,s,t),p(s,is,t),y(Q,s,t),p(s,ws,t),p(s,N,t),p(s,ds,t),y(S,s,t),p(s,ms,t),p(s,z,t),p(s,ys,t),y(H,s,t),p(s,Ms,t),p(s,K,t),Js=!0},p(s,[t]){const js={};t&2&&(js.$$scope={dirty:t,ctx:s}),j.$set(js);const ks={};t&2&&(ks.$$scope={dirty:t,ctx:s}),k.$set(ks);const Gs={};t&2&&(Gs.$$scope={dirty:t,ctx:s}),G.$set(Gs);const Bs={};t&2&&(Bs.$$scope={dirty:t,ctx:s}),B.$set(Bs);const Zs={};t&2&&(Zs.$$scope={dirty:t,ctx:s}),Z.$set(Zs)},i(s){Js||(M(e.$$.fragment,s),M(g.$$.fragment,s),M(j.$$.fragment,s),M(E.$$.fragment,s),M(k.$$.fragment,s),M(G.$$.fragment,s),M(C.$$.fragment,s),M(B.$$.fragment,s),M(Z.$$.fragment,s),M(W.$$.fragment,s),M(Y.$$.fragment,s),M(F.$$.fragment,s),M(Q.$$.fragment,s),M(S.$$.fragment,s),M(H.$$.fragment,s),Js=!0)},o(s){J(e.$$.fragment,s),J(g.$$.fragment,s),J(j.$$.fragment,s),J(E.$$.fragment,s),J(k.$$.fragment,s),J(G.$$.fragment,s),J(C.$$.fragment,s),J(B.$$.fragment,s),J(Z.$$.fragment,s),J(W.$$.fragment,s),J(Y.$$.fragment,s),J(F.$$.fragment,s),J(Q.$$.fragment,s),J(S.$$.fragment,s),J(H.$$.fragment,s),Js=!1},d(s){s&&(n(o),n(l),n(r),n(c),n(U),n(A),n(_),n(P),n($),n(L),n(q),n(O),n(ss),n(ls),n(as),n(es),n(ts),n(os),n(ns),n(V),n(ps),n(rs),n(v),n(cs),n(is),n(ws),n(N),n(ds),n(ms),n(z),n(ys),n(Ms),n(K)),n(a),T(e,s),T(g,s),T(j,s),T(E,s),T(k,s),T(G,s),T(C,s),T(B,s),T(Z,s),T(W,s),T(Y,s),T(F,s),T(Q,s),T(S,s),T(H,s)}}}const Ls='{"title":"DDP Communication Hooks","local":"ddp-communication-hooks","sections":[{"title":"FP16 Compression Hook","local":"fp16-compression-hook","sections":[{"title":"BF16 Compression Hook","local":"bf16-compression-hook","sections":[],"depth":3},{"title":"PowerSGD Hook","local":"powersgd-hook","sections":[],"depth":3}],"depth":2},{"title":"DDP Communication Hooks utilities","local":"ddp-communication-hooks-utilities","sections":[{"title":"comm_wrapper","local":"commwrapper","sections":[],"depth":3},{"title":"comm_state_option","local":"commstateoption","sections":[],"depth":3}],"depth":2}],"depth":1}';function qs(h){return gs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ol extends Es{constructor(a){super(),Cs(this,a,qs,Ps,$s,{})}}export{ol as component};
