Soofi Consortium Releases Soofi S 30B-A3B: An Open Hybrid Mamba-Transformer MoE Foundation Model For German And English

A German research consortium has published the pretraining report for Soofi S 30B-A3B. It is an open base model for German and English. Training ran end to end on Deutsche Telekom’s Industrial AI Cloud in Munich. Preview weights are on Hugging Face. It is worth noting that among some of the fully open base models tested, Soofi S records the highest English and German aggregate scores.

What is Soofi S 30B-A3B?

Soofi S is a Mixture-of-Experts (MoE) hybrid Mamba Transformer foundation model. It totals ~31.6B parameters and activates ~3.2B per token. As a base model, it has no instruction tuning, alignment, or safety tuning. The KI Bundesverband coordinates the consortium, funded by the German Federal Ministry for Economic Affairs and Energy. Participants include Fraunhofer IAIS, DFKI, TU Darmstadt, ellamind, and Merantix Momentum.

How the architecture works?

The efficiency claim starts with the layer stack. The network holds 52 layers. That is 23 Mamba-2 sequence-mixing layers, 23 granular MoE layers, and 6 Grouped-Query Attention (GQA) layers. Only those 6 GQA layers maintain a KV cache. Each MoE layer holds 128 routed experts, activates 6 per token, and adds 2 shared experts. Other details: model dimension 2688, squared ReLU, RMSNorm, and no positional embeddings.

Soofi S adopts the Nemotron 3 Nano reference design without modification. The research team gives three reasons for that choice. Those are deployability on stacks such as vLLM, serving efficiency, and scientific control. Because the backbone is fixed, Nemotron 3 Nano becomes an architecture-identical baseline. The data recipe is the only moving part.

=0) cells[cur].setAttribute(‘opacity’,’.22′); cur=(cur+1)%52; if(cur===0){ kvHit=0; kvBoxes.forEach(function(b){b.setAttribute(‘fill’,’none’);}); } cells[cur].setAttribute(‘opacity’,’1′); var k=KIND[cur]; var name = k===’m’?’Mamba-2 — recurrent state update’:(k===’e’?’MoE — 6 of 128 experts fire’:’GQA — writes to the KV cache’); l1.innerHTML=’layer ‘+(cur+1)+’ / 52 · ‘+name; if(k===’g’ && kvHit<6){ kvBoxes[kvHit].setAttribute(‘fill’,’#E10500′); kvHit++; } } document.getElementById(‘b1s’).addEventListener(‘click’,function(){ run=false; document.getElementById(‘b1’).textContent=”Play”; step(); }); document.getElementById(‘b1’).addEventListener(‘click’,function(){ run=!run; this.textContent=run?’Pause’:’Play’; }); timer=setInterval(function(){ if(run) step(); },230); step(); /* ———- pane 2: MoE routing ———- */ var svg2=document.getElementById(‘moe’); (function(){ svg2.appendChild(el(‘rect’,{x:10,y:78,width:70,height:26,rx:5,fill:’none’,stroke:’#FF8205′})); svg2.appendChild(el(‘text’,{x:45,y:95,fill:’#F2F2F2′,’font-size’:’11’,’text-anchor’:’middle’})).textContent=”token”; svg2.appendChild(el(‘rect’,{x:100,y:74,width:56,height:34,rx:5,fill:’#191919′,stroke:’#282828′})); svg2.appendChild(el(‘text’,{x:128,y:95,fill:’#8C8C8C’,’font-size’:’10’,’text-anchor’:’middle’})).textContent=”router”; svg2.appendChild(el(‘text’,{x:184,y:20,fill:’#8C8C8C’,’font-size’:’10’,’letter-spacing’:’1.2′})).textContent=”128 ROUTED EXPERTS”; })(); var exp=[]; (function(){ var cols=16,x0=184,y0=28,w=20,h=13,gx=3,gy=3; for(var i=0;i<128;i++){ var r=el(‘rect’,{x:x0+(i%cols)*(w+gx),y:y0+Math.floor(i/cols)*(h+gy),width:w,height:h,rx:2,fill:’#1e1e1e’}); svg2.appendChild(r); exp.push(r); } svg2.appendChild(el(‘text’,{x:184,y:172,fill:’#8C8C8C’,’font-size’:’10’,’letter-spacing’:’1.2′})).textContent=”2 SHARED EXPERTS (ALWAYS ON)”; svg2.appendChild(el(‘rect’,{x:400,y:161,width:34,height:13,rx:2,fill:’#FFD800′,opacity:.8})); svg2.appendChild(el(‘rect’,{x:440,y:161,width:34,height:13,rx:2,fill:’#FFD800′,opacity:.8})); })(); var l2=document.getElementById(‘l2’); function route(){ exp.forEach(function(r){ r.setAttribute(‘fill’,’#1e1e1e’); }); var pick=[],seen={}; while(pick.length<6){ var n=Math.floor(Math.random()*128); if(!seen[n]){seen[n]=1;pick.push(n);} } pick.sort(function(a,b){return a-b;}); pick.forEach(function(n,i){ setTimeout(function(){ exp[n].setAttribute(‘fill’, i<3?’#FF8205′:’#E10500′); }, reduce?0:i*90); }); l2.innerHTML=’selected: ‘+pick.map(function(n){return ‘#’+(n+1);}).join(‘ ‘); } document.getElementById(‘b2’).addEventListener(‘click’,route); route(); /* ———- pane 3: memory ———- */ var CTX=[4096,8192,16384,32768,65536,131072,262144]; var LBL=[‘4K’,’8K’,’16K’,’32K’,’64K’,’128K’,’256K’]; var BATCH=32, KV_TOK=6144, KV_TOK_FULL=53248, MAMBA=23*1048576; function fmt(b){ if(b<1048576) return (b/1024).toFixed(0)+’ KiB’; if(b<1073741824) return (b/1048576).toFixed(0)+’ MiB’; return (b/1073741824).toFixed(1)+’ GiB’; } var svg3=document.getElementById(‘mem’); var TICKS=[[1048576,’1 MiB’],[104857600,’100 MiB’],[10737418240,’10 GiB’],[1099511627776,’1 TiB’]]; function drawMem(idx){ while(svg3.firstChild) svg3.removeChild(svg3.firstChild); var W=560,H=180,x0=58,y0=14,h=132,w=W-x0-14; function yv(v){ var t=(Math.log(v)/Math.LN10-6)/6; return y0+h-Math.max(0,Math.min(1,t))*h; } TICKS.forEach(function(t){ var y=yv(t[0]); svg3.appendChild(el(‘line’,{x1:x0,y1:y,x2:x0+w,y2:y,stroke:’#1e1e1e’})); svg3.appendChild(el(‘text’,{x:x0-8,y:y+3,fill:’#8C8C8C’,’font-size’:’9′,’text-anchor’:’end’})).textContent=t[1]; }); var series=[ {c:’#E10500′,f:function(n){return KV_TOK_FULL*n*BATCH;}}, {c:’#FF8205′,f:function(n){return KV_TOK*n*BATCH;}}, {c:’#FFD800′,f:function(){return MAMBA*BATCH;}} ]; series.forEach(function(s2){ var d=”; CTX.forEach(function(n,i){ var x=x0+w*i/6, y=yv(s2.f(n)); d+=(i?’L’:’M’)+x.toFixed(1)+’,’+y.toFixed(1); }); svg3.appendChild(el(‘path’,{d:d,fill:’none’,stroke:s2.c,’stroke-width’:2,’stroke-linecap’:’round’})); svg3.appendChild(el(‘circle’,{cx:x0+w*idx/6,cy:yv(s2.f(CTX[idx])),r:4,fill:s2.c,stroke:’#0E0E0E’,’stroke-width’:2})); }); svg3.appendChild(el(‘line’,{x1:x0+w*idx/6,y1:y0,x2:x0+w*idx/6,y2:y0+h,stroke:’#3a3a3a’,’stroke-dasharray’:’3 3′})); LBL.forEach(function(t,i){ svg3.appendChild(el(‘text’,{x:x0+w*i/6,y:H-16,fill: i===idx?’#F2F2F2′:’#8C8C8C’,’font-size’:’9′,’text-anchor’:’middle’})).textContent=t; }); svg3.appendChild(el(‘text’,{x:x0+w/2,y:H-3,fill:’#8C8C8C’,’font-size’:’9′,’text-anchor’:’middle’})).textContent=”context length per sequence (log scale, batch 32)”; } var ctx=document.getElementById(‘ctx’); function upd(){ var i=+ctx.value; document.getElementById(‘ctxl’).textContent=LBL[i]; document.getElementById(‘s1’).textContent=fmt(KV_TOK*CTX[i]*BATCH); document.getElementById(‘s2’).textContent=fmt(KV_TOK_FULL*CTX[i]*BATCH); drawMem(i); } ctx.addEventListener(‘input’,upd); upd(); /* ———- pane 4: curriculum ———- */ var PH=[ {t:’Phase 1 · diverse pretraining (stable)’, d:’~20T consumed tokens on a quality-tiered mixture of web, code, math, reasoning, SFT, and German. Medium-quality web pools are listed but set to zero epochs, so up-sampling does the steering.’, l:’794,728 iterations · LR 1e-3 plateau · warmup 254′, m:[[‘English Web’,50.3],[‘Code’,14.6],[‘Reasoning’,10.4],[‘Academic & Wiki’,8.0],[‘German’,7.2],[‘Mathematics’,6.0],[‘SFT’,3.5]]}, {t:’Phase 2 · high-quality annealing (decay)’, d:’~6.58T consumed tokens. Lower-tier web is dropped entirely. German rises to 15.32% of the pool, more than triple the 5% multilingual share of the Nemotron 3 Nano reference mixture.’, l:’198,682 + 62,590 iterations · LR 1e-3 → 1e-5, then constant’, m:[[‘English Web’,36.4],[‘Code’,16.4],[‘German’,15.3],[‘Reasoning’,11.8],[‘SFT’,8.5],[‘Academic & Wiki’,6.4],[‘Mathematics’,5.2]]}, {t:’Phase 3 · long-context extension’, d:’~0.10T consumed tokens at a 1,048,576-token sequence length. The pool is ~188.5B tokens from ~21M documents, bucketed at 4K through 1M. SFT streams supply roughly 73% of it.’, l:’2,000 iterations · context-parallel 16 · LR 1e-5 → 1e-7′, m:[[‘Math SFT’,40.5],[‘General SFT’,16.6],[‘Code SFT’,15.5],[‘Web’,15.3],[‘Code’,5.0],[‘Mathematics’,3.6],[‘German’,3.5]]} ]; var svg4=document.getElementById(‘lr’); function drawLR(sel){ while(svg4.firstChild) svg4.removeChild(svg4.firstChild); var W=560,H=130,x0=14,y0=12,h=82,w=W-28; var bands=[[0,.75],[.75,.99],[.99,1]]; bands.forEach(function(b,i){ svg4.appendChild(el(‘rect’,{x:x0+w*b[0],y:y0,width:w*(b[1]-b[0]),height:h, fill:i===sel?’#FF8205′:’#1a1a1a’,opacity:i===sel?.16:1})); }); var d=’M’+x0+’,’+(y0+h); for(var p=0;p<=200;p++){ var t=p/200,v; if(t<0.01) v=t/0.01; else if(t<0.75) v=1; else if(t<0.99) v=Math.max(0.02,1-Math.sqrt((t-0.75)/0.24)*0.98); else v=0.02; d+=’L’+(x0+w*t).toFixed(1)+’,’+(y0+h-v*h*0.92).toFixed(1); } svg4.appendChild(el(‘path’,{d:d,fill:’none’,stroke:’#FFD800′,’stroke-width’:2})); [‘Phase 1 — stable’,’Phase 2 — decay’,’Phase 3′].forEach(function(t,i){ var x=i===2?(x0+w):(x0+w*((bands[i][0]+bands[i][1])/2)); var tx=el(‘text’,{x:x,y:H-16,fill:i===sel?’#F2F2F2′:’#8C8C8C’,’font-size’:’9′,’text-anchor’:i===2?’end’:’middle’}); tx.textContent=t.replace(‘—’,’–’); svg4.appendChild(tx); }); svg4.appendChild(el(‘text’,{x:x0,y:H-3,fill:’#8C8C8C’,’font-size’:’9′})).textContent=”Warmup–Stable–Decay learning rate, minus_sqrt decay (shape illustrative)”; } function setPhase(i){ document.getElementById(‘pht’).innerHTML=PH[i].t; document.getElementById(‘phd’).innerHTML=PH[i].d; document.getElementById(‘phl’).innerHTML=PH[i].l; var box=document.getElementById(‘mix’); box.innerHTML=”; PH[i].m.forEach(function(r,k){ var div=document.createElement(‘div’); div.className=”row”+(r[0]===’German’?’ hi’:”); div.innerHTML=’‘+r[0]+’‘+r[1].toFixed(1)+’%‘; box.appendChild(div); var f=div.querySelector(‘.bar span’); var pmx=Math.max.apply(null,PH[i].m.map(function(z){return z[1];})); setTimeout(function(){ f.style.width=(r[1]/pmx*100)+’%’; if(r[0]===’German’) f.style.background=’linear-gradient(90deg,#FFD800,#FA500F)’; }, reduce?0:60+k*55); }); drawLR(i); resize(); } [].slice.call(document.querySelectorAll(‘.ph’)).forEach(function(b){ b.addEventListener(‘click’,function(){ setPhase(+b.dataset.i); }); }); setPhase(0); /* ———- pane 5: benchmarks ———- */ var MODELS=[‘Soofi S 30B-A3B’,’Olmo 3 32B’,’Apertus 70B’,’EuroLLM 22B’,’Alia 40B’]; var B={ ‘English aggregate’:[70.1,67.3,62.4,61.2,59.0], ‘German aggregate’:[79.1,69.2,72.8,70.6,68.4], ‘HumanEval’:[73.8,63.0,30.2,39.3,23.8], ‘MBPP-DE’:[84.2,70.8,50.9,59.4,45.6], ‘GSM8K’:[86.1,80.7,65.4,25.1,65.4], ‘INCLUDE-DE’:[61.2,48.2,50.4,51.1,43.9], ‘GPQA-Diamond’:[43.4,33.3,27.3,30.3,29.8], ‘GLP-DE’:[88.8,73.0,81.2,78.2,65.4], ‘LBPP’:[31.0,32.1,6.4,10.7,8.6] }; var sel=document.getElementById(‘bsel’); Object.keys(B).forEach(function(k,i){ var b=document.createElement(‘button’); b.className=”act”; b.textContent=k; b.dataset.k=k; if(i===0){ b.style.borderColor=”#FA500F”; b.style.color=”#FFAF00″; } sel.appendChild(b); }); function drawB(k){ [].slice.call(sel.children).forEach(function(b){ var on=b.dataset.k===k; b.style.borderColor=on?’#FA500F’:”; b.style.color=on?’#FFAF00′:”; }); var box=document.getElementById(‘bars’); box.innerHTML=”; var mx=Math.max.apply(null,B[k]); MODELS.forEach(function(m,i){ var v=B[k][i], div=document.createElement(‘div’); div.className=”row”+(i===0?’ hi’:”); div.innerHTML=’‘+m+’‘+v.toFixed(1)+’‘; box.appendChild(div); var f=div.querySelector(‘.bar span’); if(i!==0) f.style.background=’#3a3a3a’; setTimeout(function(){ f.style.width=(v/mx*100)+’%’; }, reduce?0:50+i*70); }); resize(); } sel.addEventListener(‘click’,function(e){ if(e.target.dataset.k) drawB(e.target.dataset.k); }); drawB(‘English aggregate’); /* ———- auto-resize ———- */ var last=0; function resize(){ var root=document.getElementById(‘sf-root’); if(!root) return; var h=root.offsetHeight+40; if(Math.abs(h-last)>2){ last=h; try{ parent.postMessage({sfHeight:h},’*’); }catch(e){} } } window.addEventListener(‘load’,resize); window.addEventListener(‘resize’,resize); setInterval(resize,600); resize(); })();

Source link