1:HL["/_next/static/media/e4af272ccee01ff0-s.p.woff2","font",{"crossOrigin":"","type":"font/woff2"}] 2:HL["/_next/static/css/97132489b96da1d5.css","style",{"crossOrigin":""}] 0:["Y_TW_5cOL4VPb7FuqRz3I",[[["",{"children":[["slug","blog29","d"],{"children":["__PAGE__?{\"slug\":\"blog29\"}",{}]}]},"$undefined","$undefined",true],"$L3",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/97132489b96da1d5.css","precedence":"next","crossOrigin":""}]],"$L4"]]]] 5:HL["/_next/static/css/5b2728e81018a7be.css","style",{"crossOrigin":""}] 6:I[8326,["326","static/chunks/326-ead410bae2047633.js","986","static/chunks/986-f27c5a2c4d841870.js","42","static/chunks/app/%5Bslug%5D/page-419f452b4066bb25.js"],""] 7:I[6954,[],""] 8:I[7264,[],""] 3:[null,["$","html",null,{"lang":"ja","children":["$","body",null,{"className":"__className_f367f3 flex flex-col min-h-screen","children":[["$","header",null,{"className":"blog-header py-5","children":["$","div",null,{"className":"container mx-auto px-4","children":["$","div",null,{"className":"flex flex-col items-start","children":[["$","$L6",null,{"href":"/","className":"hover:no-underline","children":["$","h1",null,{"className":"text-6xl font-serif text-gray-800 mb-2 font-normal","children":"Shingoの数学ノート"}]}],["$","p",null,{"className":"text-xl text-gray-400 font-normal","children":"プログラミングと機械学習のメモ"}]]}]}]}],["$","main",null,{"className":"flex-grow","children":["$","$L7",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","template":["$","$L8",null,{}],"templateStyles":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"childProp":{"current":["$","$L7",null,{"parallelRouterKey":"children","segmentPath":["children",["slug","blog29","d"],"children"],"loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","template":["$","$L8",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":["$L9","$La",null],"segment":"__PAGE__?{\"slug\":\"blog29\"}"},"styles":[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/5b2728e81018a7be.css","precedence":"next","crossOrigin":""}]]}],"segment":["slug","blog29","d"]},"styles":[]}]}],["$","footer",null,{"className":"bg-[#DDDDDD] text-[#999999] py-8 mt-12 text-center border-t border-[#e5e5e5]","children":["$","div",null,{"className":"container mx-auto px-4","children":[["$","p",null,{"className":"mb-2","children":"© All rights reserved by Shingo Sekine."}],["$","p",null,{"children":["$","a",null,{"href":"#","className":"hover:text-blue-500 transition-colors","children":"Back to top"}]}]]}]}]]}]}],null] 4:[["$","meta","0",{"charSet":"utf-8"}],["$","title","1",{"children":"Shingoの数学ノート"}],["$","meta","2",{"name":"description","content":"プログラミング言語と機械学習のメモ"}],["$","meta","3",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"256x256"}],["$","meta","5",{"name":"next-size-adjust"}]] b:I[6180,["326","static/chunks/326-ead410bae2047633.js","986","static/chunks/986-f27c5a2c4d841870.js","42","static/chunks/app/%5Bslug%5D/page-419f452b4066bb25.js"],""] d:I[9703,["326","static/chunks/326-ead410bae2047633.js","986","static/chunks/986-f27c5a2c4d841870.js","42","static/chunks/app/%5Bslug%5D/page-419f452b4066bb25.js"],""] c:T21add,/*@jsxRuntime automatic @jsxImportSource react*/ const {Fragment: _Fragment, jsx: _jsx, jsxs: _jsxs} = arguments[0]; const {useMDXComponents: _provideComponents} = arguments[0]; function _createMdxContent(props) { const _components = Object.assign({ p: "p", a: "a", h2: "h2", ul: "ul", li: "li", strong: "strong", h3: "h3", img: "img", h4: "h4", div: "div", span: "span", math: "math", semantics: "semantics", mrow: "mrow", mi: "mi", mo: "mo", msub: "msub", mspace: "mspace", mfrac: "mfrac", msup: "msup", msqrt: "msqrt", annotation: "annotation", svg: "svg", path: "path", mn: "mn", mtext: "mtext", msubsup: "msubsup" }, _provideComponents(), props.components); return _jsxs(_Fragment, { children: [_jsx(_components.p, { children: "今回と次回で、BERTの理論と実装について紹介したいと思います。" }), "\n", _jsx(_components.p, { children: "今回は、実装のときに必要になるであろうBERTの理論を説明していきます。" }), "\n", _jsx(_components.p, { children: "BERTの雰囲気をつかめれば良いかなと思います。" }), "\n", _jsxs(_components.p, { children: ["基本的には、", _jsx(_components.a, { href: "https://arxiv.org/pdf/1810.04805.pdf", children: "BERTの元論文" }), "と ", _jsx(_components.a, { href: "https://arxiv.org/pdf/1706.03762.pdf", children: "Attention Is All You Need" }), "、 ", _jsx(_components.a, { href: "https://github.com/google-research/bert", children: "BERTの実装コード" }), "などを元に解説していきますので、 詳細を知りたい方はこちらを参考にしてください。(これらの論文の図を多用しています。)"] }), "\n", _jsx(_components.h2, { children: "BERTの概要" }), "\n", _jsx(_components.p, { children: "BERT（Bidirectional Encoder Representations from Transformers）は、言語表現を事前に学習するための新しい手法であり、様々な自然言語処理（NLP）タスクにおいて当時の最先端(SotA)の結果を得ています。" }), "\n", _jsx(_components.p, { children: "大規模なテキストコーパス（Wikipedia など）で汎用的な「言語理解」モデルを訓練し、そのモデルを気になる下流の NLP タスク（質問応答など）に使用します。" }), "\n", _jsx(_components.p, { children: "BERTは、NLPを事前に訓練するための初の教師なしの深い双方向システムです。" }), "\n", _jsxs(_components.p, { children: ["(参考:", _jsx(_components.a, { href: "https://github.com/google-research/bert", children: "https://github.com/google-research/bert" }), ")"] }), "\n", _jsx(_components.p, { children: "BERTの特徴をまとめると、以下のようになります。" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["\n", _jsx(_components.p, { children: "BERTは当時様々なタスクでSotAを更新したモデルである。" }), "\n"] }), "\n", _jsxs(_components.li, { children: ["\n", _jsx(_components.p, { children: "BERTは事前学習済みモデルであり、汎用的なモデルである。" }), "\n"] }), "\n", _jsxs(_components.li, { children: ["\n", _jsx(_components.p, { children: "BERTはWikipedia等の巨大なコーパスから教師なし学習で作成される。" }), "\n"] }), "\n", _jsxs(_components.li, { children: ["\n", _jsx(_components.p, { children: "BERTは文脈を双方向に学習するモデルである" }), "\n"] }), "\n"] }), "\n", _jsx(_components.h2, { children: "BERTの事前学習" }), "\n", _jsx(_components.p, { children: "BERTは事前学習済みモデルであり、事前学習で何を行うかが重要です。" }), "\n", _jsxs(_components.p, { children: ["BERTでは事前学習で手間をかけないよう、", _jsx(_components.strong, { children: "教師なし学習で事前学習を行います。" })] }), "\n", _jsx(_components.p, { children: "教師なし学習にすることで、大量のテキストデータさえあれば 人手でアノテーションを行うことなく学習をすることが可能となります。" }), "\n", _jsx(_components.p, { children: "BERTでは以下の2つのタスクを解きます。" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "マスクされた単語の予測(MLM)" }), "\n", _jsx(_components.li, { children: "次文予測(NSP)" }), "\n"] }), "\n", _jsx(_components.p, { children: "ちなみに、トレーニング前のコーパスは以下を使用しています。" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "BooksCorpus（800Mワード）（Zhu et al. 2015）" }), "\n", _jsx(_components.li, { children: "英語版ウィキペディア（2,500Mワード）" }), "\n"] }), "\n", _jsx(_components.h3, { children: "マスクされた単語の予測(MLM)" }), "\n", _jsx(_components.p, { children: "教師なし学習タスクの1つ目は、「マスクされた単語の予測(MLM)」です。" }), "\n", _jsx(_components.p, { children: "MLMはある単語をMASKさせて、周辺の単語からMASKした単語を予測する穴埋め問題です。" }), "\n", _jsx(_components.p, { children: "全体の15%の単語をマスクし、残りの単語の並びからマスクされた単語を予測します。" }), "\n", _jsx(_components.p, { children: "また、 pre-trainingで[MASK]トークンは使用しないため、選択された全ての単語を マスクするのではなく、マスクされた単語を以下のルールで置き換えます。" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image15.png", alt: "" }) }), "\n", _jsx(_components.h3, { children: "次文予測(NSP)" }), "\n", _jsx(_components.p, { children: "教師なし学習タスクの2つ目は、「次文予測(NSP)」です。" }), "\n", _jsx(_components.p, { children: "質問応答(QA)や自然言語推論(NLI)などのタスクは、2つの文の関係を理解することが必要です。" }), "\n", _jsx(_components.p, { children: "文の関係性を理解するモデルをトレーニングするために、 単語コーパスから生成できる、2値化された次文予測タスクの事前学習を行います。" }), "\n", _jsx(_components.p, { children: "具体的には、以下のように学習データを作成し、次文予測を行います。" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "50％でBがAの後に続く実際の次文（としてラベル付け）" }), "\n", _jsx(_components.li, { children: "50％がコーパスからのランダムな文（としてラベル付け）" }), "\n"] }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image16.png", alt: "" }) }), "\n", _jsx(_components.h2, { children: "BERTの構造" }), "\n", _jsx(_components.p, { children: "例えば感情分析の場合、BERTの全体像は以下のようになります。" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image1.png", alt: "" }) }), "\n", _jsxs(_components.p, { children: ["流れとしては、", _jsx(_components.strong, { children: "分かち書き(MeCab + Word Piece)から、Input Embeddingを経て、tranformer moduleをnum_hidden_layerの数だけぐるぐるします。 最後に、Output layerを経て出力します。" })] }), "\n", _jsx(_components.p, { children: "上から順にみていきましょう。" }), "\n", _jsx(_components.h3, { children: "MeCab + Word Piece" }), "\n", _jsx(_components.p, { children: "入力の際に分かち書きをする際、通常の形態素解析器やスペース区切り等で作成した場合、 語彙数が非常に大きくなります。" }), "\n", _jsxs(_components.p, { children: ["そこで、BERTでは", _jsx(_components.strong, { children: "Word Pieceというサブワード化手法" }), "を使用して語彙数を減らします。 これは、単語の中でさらに細かく分割し、高頻度の単語は1トークンとして扱い、 低頻度の単語はさらに細かく分割します。"] }), "\n", _jsx(_components.p, { children: "例えば、BERTの日本語学習済みモデル(cl-tohoku/bert-base-japanese-whole-word-masking)のWord Pieceで分割してみます。 (日本語版BERTの単語の分割はMeCabを使用しています。)" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image2.png", alt: "" }) }), "\n", _jsx(_components.p, { children: "カーネーションという単語が「カーネ」と「ーション」で分割されています。 このように、BERTでは語彙が登録されていない長い単語を短く区切ります。" }), "\n", _jsx(_components.h3, { children: "Input Embedding" }), "\n", _jsx(_components.p, { children: "BERTでは、Inputの先頭に[CLS]トークンを、文の終わりに[SEP]トークンを追加します。" }), "\n", _jsx(_components.p, { children: "また、Input Embeddingでは以下の3つのEmbeddingを作成し、足し合わせます。" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Token Embedding" }), "\n", _jsx(_components.li, { children: "Segment Embedding" }), "\n", _jsx(_components.li, { children: "Position Embedding" }), "\n"] }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image3.png", alt: "" }) }), "\n", _jsx(_components.p, { children: "それぞれについて解説していきます。" }), "\n", _jsx(_components.h4, { children: "Token Embedding" }), "\n", _jsx(_components.p, { children: "Token Embeddingでは、Tokenのidから特徴量を作成します。" }), "\n", _jsx(_components.p, { children: "実際の計算では、one-hot-encodingしてから重みを掛ける処理を行います (この層の仕組みを一般にEmbedding Layerと呼びます。)" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image4.png", alt: "" }) }), "\n", _jsx(_components.h4, { children: "Segment Embedding" }), "\n", _jsx(_components.p, { children: "Segment Embeddingでは、文章のセグメントを0か1で入力します。" }), "\n", _jsx(_components.p, { children: "感情分析のように、文章を分けなくても良い場合は全て0を入力します。" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image5.png", alt: "" }) }), "\n", _jsx(_components.h4, { children: "Positional Embedding" }), "\n", _jsx(_components.p, { children: "Positional Embeddingでは、単語の順番を0から順に入力します。" }), "\n", _jsx(_components.p, { children: "これにより単語がどの位置にあるかを識別できるようにします。" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image6.png", alt: "" }) }), "\n", _jsx(_components.p, { children: "実装で入力する際は、TokenのidとSegmentのidを入力することになります。(Positional Embeddingは内部で処理をしてくれます。)" }), "\n", _jsx(_components.h3, { children: "Attention Layer" }), "\n", _jsx(_components.p, { children: "BERTの肝であるAttention Layerをみていきましょう" }), "\n", _jsxs(_components.p, { children: ["BERTで使用されているのは", _jsx(_components.strong, { children: "Scaled Dot-Product Attention" }), "です。"] }), "\n", _jsx(_components.p, { children: "Q(Query), K(Key), V(Value)の3つの行列を使用して、トークン同士の影響度を考慮します。" }), "\n", _jsx(_components.p, { children: "Attention Layerを挟むことにより、同じ単語でも文章によって異なる特徴量を算出できます。" }), "\n", _jsx(_components.p, { children: "では、全て同一のInputから全結合を通してQ,K,Vを生成する。(Self Attentionという。)" }), "\n", _jsx(_components.p, { children: "公式の図は以下の通りです。" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image7.png", alt: "" }) }), "\n", _jsx(_components.p, { children: "式で書くと以下になります。" }), "\n", _jsx(_components.div, { className: "math math-display", children: _jsx(_components.span, { className: "katex-display", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", display: "block", children: _jsxs(_components.semantics, { children: [_jsxs(_components.mrow, { children: [_jsx(_components.mi, { children: "𝑄" }), _jsx(_components.mo, { children: "=" }), _jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝑛" }), _jsx(_components.mi, { children: "𝑠" }), _jsxs(_components.msub, { children: [_jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝑄" })] }), _jsx(_components.mo, { stretchy: "false", children: "(" }), _jsx(_components.mi, { children: "𝐼" }), _jsx(_components.mi, { children: "𝑛" }), _jsx(_components.mi, { children: "𝑝" }), _jsx(_components.mi, { children: "𝑢" }), _jsx(_components.mi, { children: "𝑡" }), _jsx(_components.mo, { stretchy: "false", children: ")" }), _jsx(_components.mspace, { linebreak: "newline" }), _jsx(_components.mi, { children: "𝐾" }), _jsx(_components.mo, { children: "=" }), _jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝑛" }), _jsx(_components.mi, { children: "𝑠" }), _jsxs(_components.msub, { children: [_jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝐾" })] }), _jsx(_components.mo, { stretchy: "false", children: "(" }), _jsx(_components.mi, { children: "𝐼" }), _jsx(_components.mi, { children: "𝑛" }), _jsx(_components.mi, { children: "𝑝" }), _jsx(_components.mi, { children: "𝑢" }), _jsx(_components.mi, { children: "𝑡" }), _jsx(_components.mo, { stretchy: "false", children: ")" }), _jsx(_components.mspace, { linebreak: "newline" }), _jsx(_components.mi, { children: "𝑉" }), _jsx(_components.mo, { children: "=" }), _jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝑛" }), _jsx(_components.mi, { children: "𝑠" }), _jsxs(_components.msub, { children: [_jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝑉" })] }), _jsx(_components.mo, { stretchy: "false", children: "(" }), _jsx(_components.mi, { children: "𝐼" }), _jsx(_components.mi, { children: "𝑛" }), _jsx(_components.mi, { children: "𝑝" }), _jsx(_components.mi, { children: "𝑢" }), _jsx(_components.mi, { children: "𝑡" }), _jsx(_components.mo, { stretchy: "false", children: ")" }), _jsx(_components.mspace, { linebreak: "newline" }), _jsx(_components.mi, { children: "A" }), _jsx(_components.mi, { children: "t" }), _jsx(_components.mi, { children: "t" }), _jsx(_components.mi, { children: "e" }), _jsx(_components.mi, { children: "n" }), _jsx(_components.mi, { children: "t" }), _jsx(_components.mi, { children: "i" }), _jsx(_components.mi, { children: "o" }), _jsx(_components.mi, { children: "n" }), _jsx(_components.mo, { stretchy: "false", children: "(" }), _jsx(_components.mi, { children: "Q" }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mi, { children: "K" }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mi, { children: "V" }), _jsx(_components.mo, { stretchy: "false", children: ")" }), _jsx(_components.mo, { children: "=" }), _jsx(_components.mi, { children: "𝑠" }), _jsx(_components.mi, { children: "𝑜" }), _jsx(_components.mi, { children: "𝑓" }), _jsx(_components.mi, { children: "𝑡" }), _jsx(_components.mi, { children: "𝑚" }), _jsx(_components.mi, { children: "𝑎" }), _jsx(_components.mi, { children: "𝑥" }), _jsx(_components.mo, { stretchy: "false", children: "(" }), _jsxs(_components.mfrac, { children: [_jsxs(_components.mrow, { children: [_jsx(_components.mi, { children: "𝑄" }), _jsxs(_components.msup, { children: [_jsx(_components.mi, { children: "𝐾" }), _jsx(_components.mi, { children: "𝑇" })] })] }), _jsx(_components.msqrt, { children: _jsx(_components.mi, { children: "𝑑" }) })] }), _jsx(_components.mo, { stretchy: "false", children: ")" }), _jsx(_components.mi, { children: "𝑉" })] }), _jsx(_components.annotation, { encoding: "application/x-tex", children: " 𝑄=𝑑𝑒𝑛𝑠𝑒_𝑄 (𝐼𝑛𝑝𝑢𝑡)\\\\ 𝐾=𝑑𝑒𝑛𝑠𝑒_𝐾 (𝐼𝑛𝑝𝑢𝑡)\\\\ 𝑉=𝑑𝑒𝑛𝑠𝑒_𝑉 (𝐼𝑛𝑝𝑢𝑡)\\\\ Attention(Q,K,V) = 𝑠𝑜𝑓𝑡𝑚𝑎𝑥 (\\frac{𝑄𝐾^𝑇}{\\sqrt{𝑑}})𝑉 " })] }) }) }), _jsxs(_components.span, { className: "katex-html", "aria-hidden": "true", children: [_jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.8778em", verticalAlign: "-0.1944em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "Q" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } }), _jsx(_components.span, { className: "mrel", children: "=" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } })] }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1.0361em", verticalAlign: "-0.2861em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "d" }), _jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mord mathnormal", children: "s" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsx(_components.span, { className: "vlist", style: { height: "0.3283em" }, children: _jsxs(_components.span, { style: { top: "-2.55em", marginLeft: "0em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "Q" }) })] }) }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.2861em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mopen", children: "(" }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.07847em" }, children: "I" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mord mathnormal", children: "p" }), _jsx(_components.span, { className: "mord mathnormal", children: "u" }), _jsx(_components.span, { className: "mord mathnormal", children: "t" }), _jsx(_components.span, { className: "mclose", children: ")" })] }), _jsx(_components.span, { className: "mspace newline" }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.6833em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.07153em" }, children: "K" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } }), _jsx(_components.span, { className: "mrel", children: "=" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } })] }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1em", verticalAlign: "-0.25em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "d" }), _jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mord mathnormal", children: "s" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsx(_components.span, { className: "vlist", style: { height: "0.3283em" }, children: _jsxs(_components.span, { style: { top: "-2.55em", marginLeft: "0em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.07153em" }, children: "K" }) })] }) }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.15em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mopen", children: "(" }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.07847em" }, children: "I" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mord mathnormal", children: "p" }), _jsx(_components.span, { className: "mord mathnormal", children: "u" }), _jsx(_components.span, { className: "mord mathnormal", children: "t" }), _jsx(_components.span, { className: "mclose", children: ")" })] }), _jsx(_components.span, { className: "mspace newline" }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.6833em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.22222em" }, children: "V" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } }), _jsx(_components.span, { className: "mrel", children: "=" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } })] }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1em", verticalAlign: "-0.25em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "d" }), _jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mord mathnormal", children: "s" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsx(_components.span, { className: "vlist", style: { height: "0.3283em" }, children: _jsxs(_components.span, { style: { top: "-2.55em", marginLeft: "0em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.22222em" }, children: "V" }) })] }) }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.15em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mopen", children: "(" }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.07847em" }, children: "I" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mord mathnormal", children: "p" }), _jsx(_components.span, { className: "mord mathnormal", children: "u" }), _jsx(_components.span, { className: "mord mathnormal", children: "t" }), _jsx(_components.span, { className: "mclose", children: ")" })] }), _jsx(_components.span, { className: "mspace newline" }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1em", verticalAlign: "-0.25em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "A" }), _jsx(_components.span, { className: "mord mathnormal", children: "tt" }), _jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mord mathnormal", children: "t" }), _jsx(_components.span, { className: "mord mathnormal", children: "i" }), _jsx(_components.span, { className: "mord mathnormal", children: "o" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mopen", children: "(" }), _jsx(_components.span, { className: "mord mathnormal", children: "Q" }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.07153em" }, children: "K" }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.22222em" }, children: "V" }), _jsx(_components.span, { className: "mclose", children: ")" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } }), _jsx(_components.span, { className: "mrel", children: "=" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } })] }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "2.4483em", verticalAlign: "-0.93em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "so" }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.10764em" }, children: "f" }), _jsx(_components.span, { className: "mord mathnormal", children: "t" }), _jsx(_components.span, { className: "mord mathnormal", children: "ma" }), _jsx(_components.span, { className: "mord mathnormal", children: "x" }), _jsx(_components.span, { className: "mopen", children: "(" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mopen nulldelimiter" }), _jsx(_components.span, { className: "mfrac", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsxs(_components.span, { className: "vlist", style: { height: "1.5183em" }, children: [_jsxs(_components.span, { style: { top: "-2.1778em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "3em" } }), _jsx(_components.span, { className: "mord", children: _jsx(_components.span, { className: "mord sqrt", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsxs(_components.span, { className: "vlist", style: { height: "0.9322em" }, children: [_jsxs(_components.span, { className: "svg-align", style: { top: "-3em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "3em" } }), _jsx(_components.span, { className: "mord", style: { paddingLeft: "0.833em" }, children: _jsx(_components.span, { className: "mord mathnormal", children: "d" }) })] }), _jsxs(_components.span, { style: { top: "-2.8922em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "3em" } }), _jsx(_components.span, { className: "hide-tail", style: { minWidth: "0.853em", height: "1.08em" }, children: _jsx(_components.svg, { xmlns: "http://www.w3.org/2000/svg", width: "400em", height: "1.08em", viewBox: "0 0 400000 1080", preserveAspectRatio: "xMinYMin slice", children: _jsx(_components.path, { d: "M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z" }) }) })] })] }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.1078em" }, children: _jsx(_components.span, {}) }) })] }) }) })] }), _jsxs(_components.span, { style: { top: "-3.23em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "3em" } }), _jsx(_components.span, { className: "frac-line", style: { borderBottomWidth: "0.04em" } })] }), _jsxs(_components.span, { style: { top: "-3.677em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "3em" } }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", children: "Q" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.07153em" }, children: "K" }), _jsx(_components.span, { className: "msupsub", children: _jsx(_components.span, { className: "vlist-t", children: _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.8413em" }, children: _jsxs(_components.span, { style: { top: "-3.063em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.13889em" }, children: "T" }) })] }) }) }) }) })] })] })] })] }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.93em" }, children: _jsx(_components.span, {}) }) })] }) }), _jsx(_components.span, { className: "mclose nulldelimiter" })] }), _jsx(_components.span, { className: "mclose", children: ")" }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.22222em" }, children: "V" })] })] })] }) }) }), "\n", _jsxs(_components.p, { children: ["ただし、", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsxs(_components.mrow, { children: [_jsx(_components.mi, { children: "Q" }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mi, { children: "K" }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mi, { children: "V" })] }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "Q, K, V" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.8778em", verticalAlign: "-0.1944em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "Q" }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.07153em" }, children: "K" }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.22222em" }, children: "V" })] }) })] }) }), "は全て(単語数×隠れ層の次元数)の行列"] }), "\n", _jsx(_components.p, { children: "さて、一つずつ解説していきましょう。" }), "\n", _jsx(_components.p, { children: "まずは、MatMulとScaleの部分です。簡単に言えば、QとKの内積を計算してスケーリング することで、QとKの類似度を算出しているようなイメージです。" }), "\n", _jsx(_components.p, { children: "なお、dim1…dimdは、d次元の特徴量を表しています。(BERT baseではd=768)" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image8.png", alt: "" }) }), "\n", _jsx(_components.p, { children: "次に、Softmaxの部分です。" }), "\n", _jsx(_components.p, { children: "これは横方向にsoftmaxを取ることで、横を足し合わせて1になるように正規化しています。" }), "\n", _jsx(_components.p, { children: "この行列から、各トークンの、周辺のトークンによる重みを可視化することができます。" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image9.png", alt: "" }) }), "\n", _jsx(_components.p, { children: "実装時、MASKは入力として与える必要があります。" }), "\n", _jsx(_components.p, { children: "最後に、SoftmaxとVのMatMulです。" }), "\n", _jsx(_components.p, { children: "これは前述のSoftmaxを使って、Vの加重平均を出しています。" }), "\n", _jsx(_components.p, { children: "つまり、Attentionでは周辺のトークンの重みを考慮して、特徴量を生成していることになります。" }), "\n", _jsx(_components.p, { children: "これにより、同じトークンでも文章によって異なった特徴量を生成することが可能です。" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image10.png", alt: "" }) }), "\n", _jsx(_components.h4, { children: "Multi-Head Attention" }), "\n", _jsx(_components.p, { children: "BERTでは、1つのAttentionではなく、特徴量の次元をh等分して複数のAttention Layerを 生成しています。これをMulti-Head Attentionと呼びます。(BERT-baseでは768次元を12等分しています。)" }), "\n", _jsx(_components.p, { children: "これにより、メモリの消費量も少なくなり、高速化も見込めます。" }), "\n", _jsx(_components.div, { className: "math math-display", children: _jsx(_components.span, { className: "katex-display", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", display: "block", children: _jsxs(_components.semantics, { children: [_jsxs(_components.mrow, { children: [_jsx(_components.mi, { children: "𝑀" }), _jsx(_components.mi, { children: "𝑢" }), _jsx(_components.mi, { children: "𝑙" }), _jsx(_components.mi, { children: "𝑡" }), _jsx(_components.mi, { children: "𝑖" }), _jsx(_components.mi, { children: "𝐻" }), _jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝑎" }), _jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mo, { stretchy: "false", children: "(" }), _jsx(_components.mi, { children: "𝑄" }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mi, { children: "𝐾" }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mi, { children: "𝑉" }), _jsx(_components.mo, { stretchy: "false", children: ")" }), _jsx(_components.mo, { children: "=" }), _jsx(_components.mi, { children: "𝐶" }), _jsx(_components.mi, { children: "𝑜" }), _jsx(_components.mi, { children: "𝑛" }), _jsx(_components.mi, { children: "𝑐" }), _jsx(_components.mi, { children: "𝑎" }), _jsx(_components.mi, { children: "𝑡" }), _jsx(_components.mo, { stretchy: "false", children: "(" }), _jsx(_components.mi, { children: "h" }), _jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝑎" }), _jsxs(_components.msub, { children: [_jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mn, { children: "1" })] }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mo, { children: "…" }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mi, { children: "h" }), _jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝑎" }), _jsxs(_components.msub, { children: [_jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mi, { children: "h" })] }), _jsx(_components.mo, { stretchy: "false", children: ")" }), _jsxs(_components.msub, { children: [_jsx(_components.mi, { children: "𝑊" }), _jsx(_components.mi, { children: "𝑂" })] }), _jsx(_components.mspace, { linebreak: "newline" }), _jsx(_components.mtext, { children: "where" }), _jsx(_components.mspace, { width: "1em" }), _jsx(_components.mi, { children: "h" }), _jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝑎" }), _jsxs(_components.msub, { children: [_jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mi, { children: "𝑖" })] }), _jsx(_components.mo, { children: "=" }), _jsx(_components.mi, { children: "𝐴" }), _jsx(_components.mi, { children: "𝑡" }), _jsx(_components.mi, { children: "𝑡" }), _jsx(_components.mi, { children: "𝑒" }), _jsx(_components.mi, { children: "𝑛" }), _jsx(_components.mi, { children: "𝑡" }), _jsx(_components.mi, { children: "𝑖" }), _jsx(_components.mi, { children: "𝑜" }), _jsx(_components.mi, { children: "𝑛" }), _jsx(_components.mo, { stretchy: "false", children: "(" }), _jsx(_components.mi, { children: "𝑄" }), _jsxs(_components.msubsup, { children: [_jsx(_components.mi, { children: "𝑊" }), _jsx(_components.mi, { children: "𝑖" }), _jsx(_components.mi, { children: "𝑄" })] }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mi, { children: "𝐾" }), _jsxs(_components.msubsup, { children: [_jsx(_components.mi, { children: "𝑊" }), _jsx(_components.mi, { children: "𝑖" }), _jsx(_components.mi, { children: "𝐾" })] }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mi, { children: "𝑉" }), _jsxs(_components.msubsup, { children: [_jsx(_components.mi, { children: "𝑊" }), _jsx(_components.mi, { children: "𝑖" }), _jsx(_components.mi, { children: "𝑉" })] }), _jsx(_components.mo, { stretchy: "false", children: ")" }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mspace, { linebreak: "newline" }), _jsxs(_components.msubsup, { children: [_jsx(_components.mi, { children: "𝑊" }), _jsx(_components.mi, { children: "𝑖" }), _jsx(_components.mi, { children: "𝑄" })] }), _jsx(_components.mo, { children: "∈" }), _jsxs(_components.msup, { children: [_jsx(_components.mi, { children: "𝑅" }), _jsxs(_components.mrow, { children: [_jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mo, { children: "×" }), _jsxs(_components.msub, { children: [_jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mi, { children: "𝑘" })] })] })] }), _jsx(_components.mo, { separator: "true", children: "," }), _jsxs(_components.msubsup, { children: [_jsx(_components.mi, { children: "𝑊" }), _jsx(_components.mi, { children: "𝑖" }), _jsx(_components.mi, { children: "𝐾" })] }), _jsx(_components.mo, { children: "∈" }), _jsxs(_components.msup, { children: [_jsx(_components.mi, { children: "𝑅" }), _jsxs(_components.mrow, { children: [_jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mo, { children: "×" }), _jsxs(_components.msub, { children: [_jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mi, { children: "𝑘" })] })] })] }), _jsx(_components.mo, { separator: "true", children: "," }), _jsxs(_components.msubsup, { children: [_jsx(_components.mi, { children: "𝑊" }), _jsx(_components.mi, { children: "𝑖" }), _jsx(_components.mi, { children: "𝑉" })] }), _jsx(_components.mo, { children: "∈" }), _jsxs(_components.msup, { children: [_jsx(_components.mi, { children: "𝑅" }), _jsxs(_components.mrow, { children: [_jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mo, { children: "×" }), _jsxs(_components.msub, { children: [_jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mi, { children: "𝑘" })] })] })] }), _jsx(_components.mo, { separator: "true", children: "," }), _jsx(_components.mi, { children: "𝑎" }), _jsx(_components.mi, { children: "𝑛" }), _jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mtext, { children: " " }), _jsxs(_components.msup, { children: [_jsx(_components.mi, { children: "𝑊" }), _jsx(_components.mi, { children: "𝑂" })] }), _jsx(_components.mo, { children: "∈" }), _jsxs(_components.msup, { children: [_jsx(_components.mi, { children: "𝑅" }), _jsxs(_components.mrow, { children: [_jsx(_components.mi, { children: "𝑑" }), _jsx(_components.mo, { children: "×" }), _jsx(_components.mi, { children: "𝑑" })] })] })] }), _jsx(_components.annotation, { encoding: "application/x-tex", children: " 𝑀𝑢𝑙𝑡𝑖𝐻𝑒𝑎𝑑(𝑄, 𝐾, 𝑉 )= 𝐶𝑜𝑛𝑐𝑎𝑡(ℎ𝑒𝑎𝑑_1, …, ℎ𝑒𝑎𝑑_ℎ ) 𝑊_𝑂 \\\\ \\text{where} \\quad ℎ𝑒𝑎𝑑_𝑖 = 𝐴𝑡𝑡𝑒𝑛𝑡𝑖𝑜𝑛(𝑄𝑊_𝑖^𝑄 , 𝐾𝑊_𝑖^𝐾 , 𝑉𝑊_𝑖^𝑉 ),\\\\ 𝑊_𝑖^𝑄 ∈ 𝑅^{𝑑×𝑑_𝑘} ,𝑊_𝑖^𝐾 ∈ 𝑅^{𝑑×𝑑_𝑘},𝑊_𝑖^𝑉 ∈ 𝑅^{𝑑×𝑑_𝑘},𝑎𝑛𝑑\\ 𝑊^𝑂 \\in 𝑅^{𝑑×𝑑} " })] }) }) }), _jsxs(_components.span, { className: "katex-html", "aria-hidden": "true", children: [_jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1em", verticalAlign: "-0.25em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.10903em" }, children: "M" }), _jsx(_components.span, { className: "mord mathnormal", children: "u" }), _jsx(_components.span, { className: "mord mathnormal", children: "lt" }), _jsx(_components.span, { className: "mord mathnormal", children: "i" }), _jsx(_components.span, { className: "mord mathnormal", children: "He" }), _jsx(_components.span, { className: "mord mathnormal", children: "a" }), _jsx(_components.span, { className: "mord mathnormal", children: "d" }), _jsx(_components.span, { className: "mopen", children: "(" }), _jsx(_components.span, { className: "mord mathnormal", children: "Q" }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.07153em" }, children: "K" }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.22222em" }, children: "V" }), _jsx(_components.span, { className: "mclose", children: ")" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } }), _jsx(_components.span, { className: "mrel", children: "=" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } })] }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1em", verticalAlign: "-0.25em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.07153em" }, children: "C" }), _jsx(_components.span, { className: "mord mathnormal", children: "o" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mord mathnormal", children: "c" }), _jsx(_components.span, { className: "mord mathnormal", children: "a" }), _jsx(_components.span, { className: "mord mathnormal", children: "t" }), _jsx(_components.span, { className: "mopen", children: "(" }), _jsx(_components.span, { className: "mord mathnormal", children: "h" }), _jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "mord mathnormal", children: "a" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", children: "d" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsx(_components.span, { className: "vlist", style: { height: "0.3011em" }, children: _jsxs(_components.span, { style: { top: "-2.55em", marginLeft: "0em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mtight", children: "1" }) })] }) }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.15em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "minner", children: "…" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "h" }), _jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "mord mathnormal", children: "a" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", children: "d" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsx(_components.span, { className: "vlist", style: { height: "0.3361em" }, children: _jsxs(_components.span, { style: { top: "-2.55em", marginLeft: "0em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "h" }) })] }) }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.15em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mclose", children: ")" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.13889em" }, children: "W" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsx(_components.span, { className: "vlist", style: { height: "0.3283em" }, children: _jsxs(_components.span, { style: { top: "-2.55em", marginLeft: "-0.1389em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.02778em" }, children: "O" }) })] }) }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.15em" }, children: _jsx(_components.span, {}) }) })] }) })] })] }), _jsx(_components.span, { className: "mspace newline" }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.8444em", verticalAlign: "-0.15em" } }), _jsx(_components.span, { className: "mord text", children: _jsx(_components.span, { className: "mord", children: "where" }) }), _jsx(_components.span, { className: "mspace", style: { marginRight: "1em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "h" }), _jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "mord mathnormal", children: "a" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", children: "d" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsx(_components.span, { className: "vlist", style: { height: "0.3117em" }, children: _jsxs(_components.span, { style: { top: "-2.55em", marginLeft: "0em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "i" }) })] }) }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.15em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } }), _jsx(_components.span, { className: "mrel", children: "=" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } })] }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1.2361em", verticalAlign: "-0.2769em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "A" }), _jsx(_components.span, { className: "mord mathnormal", children: "tt" }), _jsx(_components.span, { className: "mord mathnormal", children: "e" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mord mathnormal", children: "t" }), _jsx(_components.span, { className: "mord mathnormal", children: "i" }), _jsx(_components.span, { className: "mord mathnormal", children: "o" }), _jsx(_components.span, { className: "mord mathnormal", children: "n" }), _jsx(_components.span, { className: "mopen", children: "(" }), _jsx(_components.span, { className: "mord mathnormal", children: "Q" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.13889em" }, children: "W" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsxs(_components.span, { className: "vlist", style: { height: "0.9592em" }, children: [_jsxs(_components.span, { style: { top: "-2.4231em", marginLeft: "-0.1389em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "i" }) })] }), _jsxs(_components.span, { style: { top: "-3.1809em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "Q" }) })] })] }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.2769em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.07153em" }, children: "K" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.13889em" }, children: "W" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsxs(_components.span, { className: "vlist", style: { height: "0.8913em" }, children: [_jsxs(_components.span, { style: { top: "-2.453em", marginLeft: "-0.1389em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "i" }) })] }), _jsxs(_components.span, { style: { top: "-3.113em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.07153em" }, children: "K" }) })] })] }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.247em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.22222em" }, children: "V" }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.13889em" }, children: "W" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsxs(_components.span, { className: "vlist", style: { height: "0.8913em" }, children: [_jsxs(_components.span, { style: { top: "-2.453em", marginLeft: "-0.1389em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "i" }) })] }), _jsxs(_components.span, { style: { top: "-3.113em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.22222em" }, children: "V" }) })] })] }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.247em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mclose", children: ")" }), _jsx(_components.span, { className: "mpunct", children: "," })] }), _jsx(_components.span, { className: "mspace newline" }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1.2361em", verticalAlign: "-0.2769em" } }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.13889em" }, children: "W" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsxs(_components.span, { className: "vlist", style: { height: "0.9592em" }, children: [_jsxs(_components.span, { style: { top: "-2.4231em", marginLeft: "-0.1389em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "i" }) })] }), _jsxs(_components.span, { style: { top: "-3.1809em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "Q" }) })] })] }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.2769em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } }), _jsx(_components.span, { className: "mrel", children: "∈" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } })] }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1.1461em", verticalAlign: "-0.247em" } }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.00773em" }, children: "R" }), _jsx(_components.span, { className: "msupsub", children: _jsx(_components.span, { className: "vlist-t", children: _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.8991em" }, children: _jsxs(_components.span, { style: { top: "-3.113em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsxs(_components.span, { className: "mord mtight", children: [_jsx(_components.span, { className: "mord mathnormal mtight", children: "d" }), _jsx(_components.span, { className: "mbin mtight", children: "×" }), _jsxs(_components.span, { className: "mord mtight", children: [_jsx(_components.span, { className: "mord mathnormal mtight", children: "d" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsx(_components.span, { className: "vlist", style: { height: "0.3448em" }, children: _jsxs(_components.span, { style: { top: "-2.3488em", marginLeft: "0em", marginRight: "0.0714em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.5em" } }), _jsx(_components.span, { className: "sizing reset-size3 size1 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.03148em" }, children: "k" }) })] }) }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.1512em" }, children: _jsx(_components.span, {}) }) })] }) })] })] }) })] }) }) }) }) })] }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.13889em" }, children: "W" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsxs(_components.span, { className: "vlist", style: { height: "0.8913em" }, children: [_jsxs(_components.span, { style: { top: "-2.453em", marginLeft: "-0.1389em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "i" }) })] }), _jsxs(_components.span, { style: { top: "-3.113em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.07153em" }, children: "K" }) })] })] }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.247em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } }), _jsx(_components.span, { className: "mrel", children: "∈" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } })] }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1.1461em", verticalAlign: "-0.247em" } }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.00773em" }, children: "R" }), _jsx(_components.span, { className: "msupsub", children: _jsx(_components.span, { className: "vlist-t", children: _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.8991em" }, children: _jsxs(_components.span, { style: { top: "-3.113em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsxs(_components.span, { className: "mord mtight", children: [_jsx(_components.span, { className: "mord mathnormal mtight", children: "d" }), _jsx(_components.span, { className: "mbin mtight", children: "×" }), _jsxs(_components.span, { className: "mord mtight", children: [_jsx(_components.span, { className: "mord mathnormal mtight", children: "d" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsx(_components.span, { className: "vlist", style: { height: "0.3448em" }, children: _jsxs(_components.span, { style: { top: "-2.3488em", marginLeft: "0em", marginRight: "0.0714em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.5em" } }), _jsx(_components.span, { className: "sizing reset-size3 size1 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.03148em" }, children: "k" }) })] }) }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.1512em" }, children: _jsx(_components.span, {}) }) })] }) })] })] }) })] }) }) }) }) })] }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.13889em" }, children: "W" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsxs(_components.span, { className: "vlist", style: { height: "0.8913em" }, children: [_jsxs(_components.span, { style: { top: "-2.453em", marginLeft: "-0.1389em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", children: "i" }) })] }), _jsxs(_components.span, { style: { top: "-3.113em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.22222em" }, children: "V" }) })] })] }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.247em" }, children: _jsx(_components.span, {}) }) })] }) })] }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } }), _jsx(_components.span, { className: "mrel", children: "∈" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } })] }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "1.0935em", verticalAlign: "-0.1944em" } }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.00773em" }, children: "R" }), _jsx(_components.span, { className: "msupsub", children: _jsx(_components.span, { className: "vlist-t", children: _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.8991em" }, children: _jsxs(_components.span, { style: { top: "-3.113em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsxs(_components.span, { className: "mord mtight", children: [_jsx(_components.span, { className: "mord mathnormal mtight", children: "d" }), _jsx(_components.span, { className: "mbin mtight", children: "×" }), _jsxs(_components.span, { className: "mord mtight", children: [_jsx(_components.span, { className: "mord mathnormal mtight", children: "d" }), _jsx(_components.span, { className: "msupsub", children: _jsxs(_components.span, { className: "vlist-t vlist-t2", children: [_jsxs(_components.span, { className: "vlist-r", children: [_jsx(_components.span, { className: "vlist", style: { height: "0.3448em" }, children: _jsxs(_components.span, { style: { top: "-2.3488em", marginLeft: "0em", marginRight: "0.0714em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.5em" } }), _jsx(_components.span, { className: "sizing reset-size3 size1 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.03148em" }, children: "k" }) })] }) }), _jsx(_components.span, { className: "vlist-s", children: "​" })] }), _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.1512em" }, children: _jsx(_components.span, {}) }) })] }) })] })] }) })] }) }) }) }) })] }), _jsx(_components.span, { className: "mpunct", children: "," }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.1667em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "an" }), _jsx(_components.span, { className: "mord mathnormal", children: "d" }), _jsx(_components.span, { className: "mspace", children: " " }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.13889em" }, children: "W" }), _jsx(_components.span, { className: "msupsub", children: _jsx(_components.span, { className: "vlist-t", children: _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.8913em" }, children: _jsxs(_components.span, { style: { top: "-3.113em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mathnormal mtight", style: { marginRight: "0.02778em" }, children: "O" }) })] }) }) }) }) })] }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } }), _jsx(_components.span, { className: "mrel", children: "∈" }), _jsx(_components.span, { className: "mspace", style: { marginRight: "0.2778em" } })] }), _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.8991em" } }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.00773em" }, children: "R" }), _jsx(_components.span, { className: "msupsub", children: _jsx(_components.span, { className: "vlist-t", children: _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.8991em" }, children: _jsxs(_components.span, { style: { top: "-3.113em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsxs(_components.span, { className: "mord mtight", children: [_jsx(_components.span, { className: "mord mathnormal mtight", children: "d" }), _jsx(_components.span, { className: "mbin mtight", children: "×" }), _jsx(_components.span, { className: "mord mathnormal mtight", children: "d" })] }) })] }) }) }) }) })] })] })] })] }) }) }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image11.png", alt: "" }) }), "\n", _jsx(_components.h3, { children: "Intermediate Layer" }), "\n", _jsx(_components.p, { children: "Intermediate LayerはFeed Forword Networkとも呼ばれ、トークンごとに全結合を行う層です。" }), "\n", _jsx(_components.p, { children: "BERT baseでは、768次元を3072次元まで広げた後に、また768次元に戻しています。" }), "\n", _jsx(_components.p, { children: "BERTのIntermediate Layerでは活性化関数にgeluを用いています。" }), "\n", _jsx(_components.p, { children: "geluはreluに近い関数ですが、0付近で滑らかになっていて、実数全体で微分可能です。" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image12.png", alt: "" }) }), "\n", _jsx(_components.h3, { children: "Output Layer" }), "\n", _jsx(_components.p, { children: "Output layerはタスクによって異なります。" }), "\n", _jsx(_components.p, { children: "次文予測(NSP)や感情分析等の文章ごとのクラス分類では、 [CLS]トークンの特徴量を使用して全結合を行います。" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image13.png", alt: "" }) }), "\n", _jsx(_components.p, { children: "マスクされた単語予測(MLM)やQA、固有表現抽出などの単語ごとに出力する場合は、 トークンごと全結合を行い結果を出力します。" }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "/images/blog29/image14.png", alt: "" }) }), "\n", _jsx(_components.h2, { children: "まとめ" }), "\n", _jsx(_components.p, { children: "BERTの実装のための理解にあたって、必要な箇所をまとめました。" }), "\n", _jsx(_components.p, { children: "ブログに書ききれていない部分も多々あると思いますが、BERTに関する素晴らしい論文やブログなどはたくさんありますのでぜひ他も調べてみてください。" }), "\n", _jsx(_components.p, { children: "この次のブログで、実装コードを紹介していく予定です。" }), "\n", _jsx(_components.h2, { children: "参考文献" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://booth.pm/ja/items/1834866", children: "BERT・XLNet に学ぶ、言語処理における事前学習 lib-arts 著" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://arxiv.org/pdf/1810.04805.pdf", children: "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding.\u000bJacob Devlin, Ming-Wei Chang, Kenton Lee, Kristina Toutanova著" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://arxiv.org/pdf/1706.03762.pdf", children: "Attention Is All You Need. Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, \u000bLlion Jones, Aidan N. Gomez, Lukasz Kaiser, Illia Polosukhin著" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://arxiv.org/pdf/1906.05714.pdf", children: "A Multiscale Visualization of Attention in the Transformer Model. Jesse Vig著" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://medium.com/@shoray.goel/gelu-gaussian-error-linear-unit-4ec59fb2e47c", children: "GELU activation" }) }), "\n", _jsx(_components.li, { children: "つくりながら学ぶ! PyTorchによる発展ディープラーニング 小川雄太郎 著" }), "\n"] })] }); } function MDXContent(props = {}) { const {wrapper: MDXLayout} = Object.assign({}, _provideComponents(), props.components); return MDXLayout ? _jsx(MDXLayout, Object.assign({}, props, { children: _jsx(_createMdxContent, props) })) : _createMdxContent(props); } return { default: MDXContent }; a:["$","div",null,{"className":"container mx-auto px-4 py-8 max-w-7xl","children":["$","div",null,{"className":"flex flex-col md:flex-row gap-8","children":[["$","article",null,{"className":"w-full md:w-[70%] bg-white shadow-lg rounded-lg p-8","children":[["$","header",null,{"className":"mb-8 border-b pb-4","children":[["$","div",null,{"className":"flex items-center gap-2 mb-2","children":[["$","$L6",null,{"href":"/","className":"text-gray-500 hover:text-orange-500","children":"Home"}],["$","span",null,{"className":"text-gray-300","children":">"}],["$","span",null,{"className":"text-gray-500","children":"$undefined"}]]}],["$","h1",null,{"className":"text-3xl font-bold mb-4","children":"BERTを使ってみよう1"}],["$","div",null,{"className":"text-gray-500 flex flex-wrap gap-4 items-center mb-6","children":[["$","time",null,{"className":"flex items-center gap-1 text-sm","children":[["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","height":"18px","viewBox":"0 0 24 24","width":"18px","fill":"#666666","className":"mr-1","children":[["$","path",null,{"d":"M0 0h24v24H0V0z","fill":"none"}],["$","path",null,{"d":"M19 3h-1V1h-2v2H8V1H6v2H5c-1.11 0-2 .9-2 2v14c0 1.1.89 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zm0 16H5V8h14v11zM7 10h5v5H7z"}]]}],"日付: ","2021-04-18"]}],["$","div",null,{"className":"flex flex-wrap gap-2","children":[["$","$L6","自然言語処理",{"href":"/tags/自然言語処理","className":"bg-gray-100 hover:bg-blue-100 px-2 py-1 rounded text-sm text-gray-600 hover:text-blue-600 transition-colors","children":["#","自然言語処理"]}]]}]]}]]}],["$","div",null,{"className":"prose max-w-none prose-headings:border-b prose-headings:pb-2 prose-a:text-orange-600","children":["$","$Lb",null,{"compiledSource":"$c","frontmatter":{},"scope":{}}]}],["$","div",null,{"className":"flex justify-between items-center mt-12","children":[["$","div",null,{"className":"flex-1","children":["$","$L6",null,{"href":"/blog28","className":"group flex flex-col items-start p-4 rounded-lg border border-gray-100 hover:border-orange-200 hover:bg-orange-50 transition-all","children":[["$","span",null,{"className":"text-sm text-gray-500 mb-1 group-hover:text-orange-600 flex items-center","children":[["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","className":"h-4 w-4 mr-1","fill":"none","viewBox":"0 0 24 24","stroke":"currentColor","children":["$","path",null,{"strokeLinecap":"round","strokeLinejoin":"round","strokeWidth":2,"d":"M15 19l-7-7 7-7"}]}],"Previous"]}],["$","span",null,{"className":"font-medium text-gray-800 dark:text-gray-200 line-clamp-2","children":"日本語ポジネガ分析の教師データ作成はYahoo!ローカルサーチAPIがおすすめって話"}]]}]}],["$","div",null,{"className":"w-8"}],["$","div",null,{"className":"flex-1 text-right","children":["$","$L6",null,{"href":"/blog30","className":"group flex flex-col items-end p-4 rounded-lg border border-gray-100 hover:border-orange-200 hover:bg-orange-50 transition-all","children":[["$","span",null,{"className":"text-sm text-gray-500 mb-1 group-hover:text-orange-600 flex items-center","children":["Next",["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","className":"h-4 w-4 ml-1","fill":"none","viewBox":"0 0 24 24","stroke":"currentColor","children":["$","path",null,{"strokeLinecap":"round","strokeLinejoin":"round","strokeWidth":2,"d":"M9 5l7 7-7 7"}]}]]}],["$","span",null,{"className":"font-medium text-gray-800 dark:text-gray-200 line-clamp-2","children":"BERTを使ってみよう2"}]]}]}]]}],["$","$Ld",null,{}]]}],["$","div",null,{"className":"w-full md:w-[30%]","children":"$Le"}]]}]}] 9:null f:I[9009,["326","static/chunks/326-ead410bae2047633.js","986","static/chunks/986-f27c5a2c4d841870.js","42","static/chunks/app/%5Bslug%5D/page-419f452b4066bb25.js"],""] e:["$","aside",null,{"className":"w-full flex flex-col gap-6","children":[["$","div",null,{"className":"bg-white p-6 rounded shadow-sm border","children":[["$","h3",null,{"className":"font-bold text-lg mb-4 text-[#494949] border-b pb-2","children":"プロフィール"}],["$","div",null,{"className":"flex items-center mb-4","children":[["$","$Lf",null,{}],["$","span",null,{"className":"font-bold","children":"Shingo.S"}]]}],["$","p",null,{"className":"mb-4 text-base leading-relaxed text-gray-600","children":["データサイエンティストとして働いています。仕事では主にPythonやSAS、Rを用いて分析しています。",["$","br",null,{}],["$","br",null,{}],"統計検定1級、Kaggle Expert。",["$","br",null,{}],"自然言語処理に興味があります。"]}],["$","div",null,{"className":"mt-4 flex items-center","children":["$","a",null,{"href":"https://x.com/shingo97358922","target":"_blank","rel":"noopener noreferrer","className":"flex items-center gap-2 text-gray-600 hover:text-black transition-colors","children":[["$","svg",null,{"className":"w-5 h-5","fill":"currentColor","viewBox":"0 0 24 24","aria-hidden":"true","children":["$","path",null,{"d":"M18.244 2.25h3.308l-7.227 8.26 8.502 11.24H16.17l-5.214-6.817L4.99 21.75H1.68l7.73-8.835L1.254 2.25H8.08l4.713 6.231zm-1.161 17.52h1.833L7.084 4.126H5.117z"}]}],["$","span",null,{"className":"text-sm font-medium","children":"@shingo97358922"}]]}]}]]}],["$","div",null,{"className":"bg-white p-6 rounded shadow-sm border","children":[["$","h3",null,{"className":"font-bold text-lg mb-4 text-[#494949] border-b pb-2","children":"アクセスカウンター"}],["$","div",null,{"className":"flex flex-col gap-2 text-gray-600 pl-2","children":[["$","div",null,{"className":"flex items-center gap-4","children":[["$","span",null,{"className":"w-12 text-right","children":"累計"}],["$","span",null,{"children":":"}],["$","span",null,{"className":"font-mono text-lg font-bold","children":"72320"}]]}],["$","div",null,{"className":"flex items-center gap-4","children":[["$","span",null,{"className":"w-12 text-right","children":"本日"}],["$","span",null,{"children":":"}],["$","span",null,{"className":"font-mono text-lg font-bold","children":"17"}]]}],["$","div",null,{"className":"flex items-center gap-4","children":[["$","span",null,{"className":"w-12 text-right","children":"昨日"}],["$","span",null,{"children":":"}],["$","span",null,{"className":"font-mono text-lg font-bold","children":"33"}]]}]]}]]}],["$","div",null,{"className":"bg-white p-6 rounded shadow-sm border","children":[["$","h3",null,{"className":"font-bold text-lg mb-4 text-[#494949] border-b pb-2","children":"月別アーカイブ"}],["$","ul",null,{"className":"text-base text-gray-600","children":[["$","li","2026年2月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2026-02","className":"hover:text-blue-500 transition-colors block","children":["2026年2月"," (",1,")"]}]}],["$","li","2025年2月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2025-02","className":"hover:text-blue-500 transition-colors block","children":["2025年2月"," (",1,")"]}]}],["$","li","2024年10月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2024-10","className":"hover:text-blue-500 transition-colors block","children":["2024年10月"," (",1,")"]}]}],["$","li","2024年7月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2024-07","className":"hover:text-blue-500 transition-colors block","children":["2024年7月"," (",1,")"]}]}],["$","li","2024年2月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2024-02","className":"hover:text-blue-500 transition-colors block","children":["2024年2月"," (",1,")"]}]}],["$","li","2023年6月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2023-06","className":"hover:text-blue-500 transition-colors block","children":["2023年6月"," (",1,")"]}]}],["$","li","2023年5月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2023-05","className":"hover:text-blue-500 transition-colors block","children":["2023年5月"," (",1,")"]}]}],["$","li","2023年3月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2023-03","className":"hover:text-blue-500 transition-colors block","children":["2023年3月"," (",2,")"]}]}],["$","li","2022年10月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2022-10","className":"hover:text-blue-500 transition-colors block","children":["2022年10月"," (",2,")"]}]}],["$","li","2022年9月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2022-09","className":"hover:text-blue-500 transition-colors block","children":["2022年9月"," (",2,")"]}]}],["$","li","2022年7月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2022-07","className":"hover:text-blue-500 transition-colors block","children":["2022年7月"," (",1,")"]}]}],["$","li","2022年3月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2022-03","className":"hover:text-blue-500 transition-colors block","children":["2022年3月"," (",1,")"]}]}],["$","li","2021年11月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2021-11","className":"hover:text-blue-500 transition-colors block","children":["2021年11月"," (",1,")"]}]}],["$","li","2021年10月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2021-10","className":"hover:text-blue-500 transition-colors block","children":["2021年10月"," (",2,")"]}]}],["$","li","2021年7月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2021-07","className":"hover:text-blue-500 transition-colors block","children":["2021年7月"," (",2,")"]}]}],["$","li","2021年5月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2021-05","className":"hover:text-blue-500 transition-colors block","children":["2021年5月"," (",2,")"]}]}],["$","li","2021年4月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2021-04","className":"hover:text-blue-500 transition-colors block","children":["2021年4月"," (",2,")"]}]}],["$","li","2021年2月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2021-02","className":"hover:text-blue-500 transition-colors block","children":["2021年2月"," (",1,")"]}]}],["$","li","2020年10月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2020-10","className":"hover:text-blue-500 transition-colors block","children":["2020年10月"," (",1,")"]}]}],["$","li","2020年9月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2020-09","className":"hover:text-blue-500 transition-colors block","children":["2020年9月"," (",1,")"]}]}],["$","li","2020年7月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2020-07","className":"hover:text-blue-500 transition-colors block","children":["2020年7月"," (",1,")"]}]}],["$","li","2020年6月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2020-06","className":"hover:text-blue-500 transition-colors block","children":["2020年6月"," (",1,")"]}]}],["$","li","2020年5月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2020-05","className":"hover:text-blue-500 transition-colors block","children":["2020年5月"," (",1,")"]}]}],["$","li","2020年4月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2020-04","className":"hover:text-blue-500 transition-colors block","children":["2020年4月"," (",1,")"]}]}],["$","li","2020年3月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2020-03","className":"hover:text-blue-500 transition-colors block","children":["2020年3月"," (",1,")"]}]}],["$","li","2020年2月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2020-02","className":"hover:text-blue-500 transition-colors block","children":["2020年2月"," (",1,")"]}]}],["$","li","2020年1月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2020-01","className":"hover:text-blue-500 transition-colors block","children":["2020年1月"," (",1,")"]}]}],["$","li","2019年12月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2019-12","className":"hover:text-blue-500 transition-colors block","children":["2019年12月"," (",1,")"]}]}],["$","li","2019年11月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2019-11","className":"hover:text-blue-500 transition-colors block","children":["2019年11月"," (",2,")"]}]}],["$","li","2019年10月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2019-10","className":"hover:text-blue-500 transition-colors block","children":["2019年10月"," (",2,")"]}]}],["$","li","2019年9月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2019-09","className":"hover:text-blue-500 transition-colors block","children":["2019年9月"," (",1,")"]}]}],["$","li","2019年8月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2019-08","className":"hover:text-blue-500 transition-colors block","children":["2019年8月"," (",8,")"]}]}],["$","li","2019年7月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2019-07","className":"hover:text-blue-500 transition-colors block","children":["2019年7月"," (",1,")"]}]}],["$","li","2019年6月",{"className":"mb-2 border-b border-dashed border-gray-200 pb-1 last:border-none","children":["$","$L6",null,{"href":"/archives/2019-06","className":"hover:text-blue-500 transition-colors block","children":["2019年6月"," (",2,")"]}]}]]}]]}],["$","div",null,{"className":"bg-white p-6 rounded shadow-sm border","children":[["$","h3",null,{"className":"font-bold text-lg mb-4 text-[#494949] border-b pb-2","children":"タグ一覧"}],["$","div",null,{"className":"flex flex-wrap gap-2","children":[["$","$L6","データ分析",{"href":"/tags/データ分析","className":"bg-gray-100 hover:bg-blue-500 hover:text-white rounded px-2 py-1 text-sm transition-colors duration-200 text-gray-600","children":["データ分析"," (",18,")"]}],["$","$L6","自然言語処理",{"href":"/tags/自然言語処理","className":"bg-gray-100 hover:bg-blue-500 hover:text-white rounded px-2 py-1 text-sm transition-colors duration-200 text-gray-600","children":["自然言語処理"," (",16,")"]}],["$","$L6","SAS",{"href":"/tags/SAS","className":"bg-gray-100 hover:bg-blue-500 hover:text-white rounded px-2 py-1 text-sm transition-colors duration-200 text-gray-600","children":["SAS"," (",12,")"]}],["$","$L6","Kaggle",{"href":"/tags/Kaggle","className":"bg-gray-100 hover:bg-blue-500 hover:text-white rounded px-2 py-1 text-sm transition-colors duration-200 text-gray-600","children":["Kaggle"," (",3,")"]}],["$","$L6","雑談",{"href":"/tags/雑談","className":"bg-gray-100 hover:bg-blue-500 hover:text-white rounded px-2 py-1 text-sm transition-colors duration-200 text-gray-600","children":["雑談"," (",2,")"]}]]}]]}],["$","div",null,{"className":"bg-white p-6 rounded shadow-sm border","children":[["$","h3",null,{"className":"font-bold text-lg mb-4 text-[#494949] border-b pb-2","children":"広告枠"}],["$","div",null,{"className":"flex justify-center items-center","children":[["$","a",null,{"href":"https://px.a8.net/svt/ejp?a8mat=4AXI0F+CULTTE+348+6CWQP","rel":"nofollow","target":"_blank","children":["$","img",null,{"style":{"border":0},"width":"250","height":"250","alt":"","src":"https://www25.a8.net/svt/bgt?aid=260223855777&wid=001&eno=01&mid=s00000000404001068000&mc=1"}]}],["$","img",null,{"style":{"border":0},"width":"1","height":"1","src":"https://www17.a8.net/0.gif?a8mat=4AXI0F+CULTTE+348+6CWQP","alt":""}]]}]]}]]}]