diff --git a/conductor/tracks/video_analysis_cs229_building_llms_20260621/artifacts/transcript.json b/conductor/tracks/video_analysis_cs229_building_llms_20260621/artifacts/transcript.json
new file mode 100644
index 00000000..9cc6e96a
--- /dev/null
+++ b/conductor/tracks/video_analysis_cs229_building_llms_20260621/artifacts/transcript.json
@@ -0,0 +1,26993 @@
+{
+  "video_id": "9vM4p9NN0Ts",
+  "segments": [
+    {
+      "start": 5.6,
+      "duration": 0.0,
+      "text": "so<00:00:05.879><c> let's</c><00:00:06.120><c> get</c><00:00:06.279><c> started</c><00:00:06.960><c> uh</c><00:00:07.080><c> so</c><00:00:07.200><c> I'll</c><00:00:07.359><c> be</c>"
+    },
+    {
+      "start": 7.51,
+      "duration": 0.0,
+      "text": "so let's get started uh so I'll be"
+    },
+    {
+      "start": 7.52,
+      "duration": 0.0,
+      "text": "so let's get started uh so I'll be talking<00:00:07.799><c> about</c><00:00:08.080><c> building</c><00:00:08.599><c> llms</c><00:00:09.200><c> today</c><00:00:09.920><c> um</c><00:00:10.120><c> so</c>"
+    },
+    {
+      "start": 10.31,
+      "duration": 0.0,
+      "text": "talking about building llms today um so"
+    },
+    {
+      "start": 10.32,
+      "duration": 0.0,
+      "text": "talking about building llms today um so I<00:00:10.400><c> think</c><00:00:10.559><c> a</c><00:00:10.679><c> lot</c><00:00:10.800><c> of</c><00:00:10.960><c> you</c><00:00:11.240><c> have</c><00:00:11.440><c> heard</c><00:00:12.160><c> of</c><00:00:12.320><c> llms</c>"
+    },
+    {
+      "start": 12.91,
+      "duration": 0.0,
+      "text": "I think a lot of you have heard of llms"
+    },
+    {
+      "start": 12.92,
+      "duration": 0.0,
+      "text": "I think a lot of you have heard of llms before<00:00:13.920><c> uh</c><00:00:14.080><c> but</c><00:00:14.240><c> just</c><00:00:14.400><c> as</c><00:00:14.519><c> a</c><00:00:14.719><c> quick</c><00:00:15.000><c> recap</c><00:00:15.920><c> uh</c>"
+    },
+    {
+      "start": 16.15,
+      "duration": 0.0,
+      "text": "before uh but just as a quick recap uh"
+    },
+    {
+      "start": 16.16,
+      "duration": 0.0,
+      "text": "before uh but just as a quick recap uh llms<00:00:16.760><c> standing</c><00:00:17.080><c> for</c><00:00:17.359><c> large</c><00:00:17.680><c> language</c><00:00:18.000><c> models</c>"
+    },
+    {
+      "start": 18.67,
+      "duration": 0.0,
+      "text": "llms standing for large language models"
+    },
+    {
+      "start": 18.68,
+      "duration": 0.0,
+      "text": "llms standing for large language models are<00:00:18.920><c> basically</c><00:00:19.279><c> all</c><00:00:19.439><c> the</c><00:00:19.560><c> chat</c><00:00:19.840><c> Bots</c><00:00:20.680><c> uh</c><00:00:20.840><c> that</c>"
+    },
+    {
+      "start": 20.95,
+      "duration": 0.0,
+      "text": "are basically all the chat Bots uh that"
+    },
+    {
+      "start": 20.96,
+      "duration": 0.0,
+      "text": "are basically all the chat Bots uh that you've<00:00:21.119><c> been</c><00:00:21.320><c> hearing</c><00:00:21.920><c> about</c><00:00:22.279><c> recently</c><00:00:22.880><c> so</c><00:00:23.800><c> uh</c>"
+    },
+    {
+      "start": 23.99,
+      "duration": 0.0,
+      "text": "you've been hearing about recently so uh"
+    },
+    {
+      "start": 24.0,
+      "duration": 0.0,
+      "text": "you've been hearing about recently so uh Chad<00:00:24.240><c> GPT</c><00:00:25.039><c> from</c><00:00:25.240><c> open</c><00:00:25.640><c> ey</c><00:00:26.320><c> Claud</c><00:00:27.199><c> from</c>"
+    },
+    {
+      "start": 27.39,
+      "duration": 0.0,
+      "text": "Chad GPT from open ey Claud from"
+    },
+    {
+      "start": 27.4,
+      "duration": 0.0,
+      "text": "Chad GPT from open ey Claud from entropic<00:00:28.039><c> Gemini</c><00:00:28.760><c> and</c><00:00:28.960><c> and</c><00:00:29.080><c> lman</c><00:00:29.679><c> other</c><00:00:30.039><c> type</c>"
+    },
+    {
+      "start": 30.189,
+      "duration": 0.0,
+      "text": "entropic Gemini and and lman other type"
+    },
+    {
+      "start": 30.199,
+      "duration": 0.0,
+      "text": "entropic Gemini and and lman other type of<00:00:30.279><c> models</c><00:00:30.599><c> like</c><00:00:30.800><c> this</c><00:00:31.400><c> and</c><00:00:31.519><c> today</c><00:00:31.759><c> we'll</c><00:00:31.960><c> be</c>"
+    },
+    {
+      "start": 32.15,
+      "duration": 0.0,
+      "text": "of models like this and today we'll be"
+    },
+    {
+      "start": 32.16,
+      "duration": 0.0,
+      "text": "of models like this and today we'll be talking<00:00:32.439><c> about</c><00:00:32.800><c> how</c><00:00:33.079><c> do</c><00:00:33.399><c> they</c><00:00:33.719><c> actually</c><00:00:34.040><c> work</c>"
+    },
+    {
+      "start": 34.59,
+      "duration": 0.0,
+      "text": "talking about how do they actually work"
+    },
+    {
+      "start": 34.6,
+      "duration": 0.0,
+      "text": "talking about how do they actually work so<00:00:34.760><c> it's</c><00:00:34.879><c> going</c><00:00:35.000><c> to</c><00:00:35.040><c> be</c><00:00:35.160><c> an</c><00:00:35.280><c> overview</c><00:00:35.680><c> because</c>"
+    },
+    {
+      "start": 35.79,
+      "duration": 0.0,
+      "text": "so it's going to be an overview because"
+    },
+    {
+      "start": 35.8,
+      "duration": 0.0,
+      "text": "so it's going to be an overview because it's<00:00:35.960><c> only</c><00:00:36.200><c> one</c><00:00:36.399><c> lecture</c><00:00:36.840><c> and</c><00:00:36.920><c> it's</c><00:00:37.079><c> hard</c><00:00:37.239><c> to</c>"
+    },
+    {
+      "start": 37.35,
+      "duration": 0.0,
+      "text": "it's only one lecture and it's hard to"
+    },
+    {
+      "start": 37.36,
+      "duration": 0.0,
+      "text": "it's only one lecture and it's hard to compress<00:00:37.760><c> everything</c><00:00:38.239><c> but</c><00:00:38.680><c> hopefully</c><00:00:39.120><c> I'll</c>"
+    },
+    {
+      "start": 39.27,
+      "duration": 0.0,
+      "text": "compress everything but hopefully I'll"
+    },
+    {
+      "start": 39.28,
+      "duration": 0.0,
+      "text": "compress everything but hopefully I'll touch<00:00:39.559><c> a</c><00:00:39.640><c> little</c><00:00:39.800><c> bit</c><00:00:40.000><c> about</c><00:00:40.280><c> all</c><00:00:40.440><c> the</c>"
+    },
+    {
+      "start": 40.549,
+      "duration": 0.0,
+      "text": "touch a little bit about all the"
+    },
+    {
+      "start": 40.559,
+      "duration": 0.0,
+      "text": "touch a little bit about all the components<00:00:40.960><c> that</c><00:00:41.039><c> are</c><00:00:41.160><c> needed</c><00:00:41.399><c> to</c><00:00:41.520><c> train</c><00:00:42.280><c> uh</c>"
+    },
+    {
+      "start": 42.389,
+      "duration": 0.0,
+      "text": "components that are needed to train uh"
+    },
+    {
+      "start": 42.399,
+      "duration": 0.0,
+      "text": "components that are needed to train uh some<00:00:42.559><c> of</c><00:00:42.719><c> these</c><00:00:43.000><c> llms</c><00:00:44.000><c> uh</c><00:00:44.120><c> also</c><00:00:44.360><c> if</c><00:00:44.480><c> you</c><00:00:44.600><c> have</c>"
+    },
+    {
+      "start": 44.79,
+      "duration": 0.0,
+      "text": "some of these llms uh also if you have"
+    },
+    {
+      "start": 44.8,
+      "duration": 0.0,
+      "text": "some of these llms uh also if you have questions<00:00:45.320><c> please</c><00:00:45.640><c> interrupt</c><00:00:46.120><c> me</c><00:00:46.399><c> and</c><00:00:46.680><c> ask</c><00:00:47.600><c> uh</c>"
+    },
+    {
+      "start": 47.709,
+      "duration": 0.0,
+      "text": "questions please interrupt me and ask uh"
+    },
+    {
+      "start": 47.719,
+      "duration": 0.0,
+      "text": "questions please interrupt me and ask uh if<00:00:47.879><c> you</c><00:00:48.039><c> have</c><00:00:48.160><c> a</c><00:00:48.360><c> question</c><00:00:48.920><c> most</c><00:00:49.239><c> likely</c><00:00:49.600><c> other</c>"
+    },
+    {
+      "start": 49.83,
+      "duration": 0.0,
+      "text": "if you have a question most likely other"
+    },
+    {
+      "start": 49.84,
+      "duration": 0.0,
+      "text": "if you have a question most likely other people<00:00:50.039><c> in</c><00:00:50.160><c> the</c><00:00:50.280><c> room</c><00:00:50.840><c> or</c><00:00:51.120><c> on</c><00:00:51.360><c> Zoom</c><00:00:52.120><c> have</c><00:00:52.359><c> other</c>"
+    },
+    {
+      "start": 52.79,
+      "duration": 0.0,
+      "text": "people in the room or on Zoom have other"
+    },
+    {
+      "start": 52.8,
+      "duration": 0.0,
+      "text": "people in the room or on Zoom have other have<00:00:52.960><c> the</c><00:00:53.079><c> same</c><00:00:53.320><c> question</c><00:00:53.719><c> so</c><00:00:54.079><c> please</c><00:00:54.800><c> ask</c><00:00:55.800><c> um</c>"
+    },
+    {
+      "start": 56.51,
+      "duration": 0.0,
+      "text": "have the same question so please ask um"
+    },
+    {
+      "start": 56.52,
+      "duration": 0.0,
+      "text": "have the same question so please ask um great<00:00:56.920><c> so</c><00:00:57.160><c> what</c><00:00:57.399><c> matters</c><00:00:57.920><c> when</c><00:00:58.079><c> training</c><00:00:58.760><c> llms</c>"
+    },
+    {
+      "start": 59.75,
+      "duration": 0.0,
+      "text": "great so what matters when training llms"
+    },
+    {
+      "start": 59.76,
+      "duration": 0.0,
+      "text": "great so what matters when training llms um<00:01:00.160><c> so</c><00:01:00.320><c> there</c><00:01:00.440><c> a</c><00:01:00.519><c> few</c><00:01:00.760><c> key</c><00:01:00.960><c> components</c><00:01:01.680><c> that</c>"
+    },
+    {
+      "start": 61.869,
+      "duration": 0.0,
+      "text": "um so there a few key components that"
+    },
+    {
+      "start": 61.879,
+      "duration": 0.0,
+      "text": "um so there a few key components that matter<00:01:02.879><c> uh</c><00:01:03.000><c> one</c><00:01:03.199><c> is</c><00:01:03.359><c> the</c><00:01:03.480><c> architecture</c><00:01:04.040><c> so</c><00:01:04.239><c> as</c>"
+    },
+    {
+      "start": 64.35,
+      "duration": 0.0,
+      "text": "matter uh one is the architecture so as"
+    },
+    {
+      "start": 64.36,
+      "duration": 0.0,
+      "text": "matter uh one is the architecture so as you<00:01:04.640><c> probably</c><00:01:04.960><c> all</c><00:01:05.119><c> know</c><00:01:05.479><c> LMS</c><00:01:06.040><c> are</c><00:01:06.240><c> newal</c>"
+    },
+    {
+      "start": 66.59,
+      "duration": 0.0,
+      "text": "you probably all know LMS are newal"
+    },
+    {
+      "start": 66.6,
+      "duration": 0.0,
+      "text": "you probably all know LMS are newal networks<00:01:07.520><c> and</c><00:01:07.759><c> when</c><00:01:08.000><c> you</c><00:01:08.240><c> think</c><00:01:08.439><c> about</c><00:01:08.640><c> new</c>"
+    },
+    {
+      "start": 68.91,
+      "duration": 0.0,
+      "text": "networks and when you think about new"
+    },
+    {
+      "start": 68.92,
+      "duration": 0.0,
+      "text": "networks and when you think about new networks<00:01:09.240><c> you</c><00:01:09.360><c> have</c><00:01:09.479><c> to</c><00:01:09.640><c> think</c><00:01:09.799><c> about</c><00:01:10.080><c> what</c>"
+    },
+    {
+      "start": 70.23,
+      "duration": 0.0,
+      "text": "networks you have to think about what"
+    },
+    {
+      "start": 70.24,
+      "duration": 0.0,
+      "text": "networks you have to think about what architecture<00:01:10.720><c> you're</c><00:01:10.880><c> using</c><00:01:11.720><c> and</c><00:01:11.880><c> another</c>"
+    },
+    {
+      "start": 72.149,
+      "duration": 0.0,
+      "text": "architecture you're using and another"
+    },
+    {
+      "start": 72.159,
+      "duration": 0.0,
+      "text": "architecture you're using and another component<00:01:12.520><c> which</c><00:01:12.640><c> is</c><00:01:12.759><c> really</c><00:01:13.000><c> important</c><00:01:13.840><c> uh</c>"
+    },
+    {
+      "start": 73.91,
+      "duration": 0.0,
+      "text": "component which is really important uh"
+    },
+    {
+      "start": 73.92,
+      "duration": 0.0,
+      "text": "component which is really important uh is<00:01:14.000><c> the</c><00:01:14.159><c> training</c><00:01:14.439><c> loss</c><00:01:14.799><c> and</c><00:01:14.880><c> the</c><00:01:15.000><c> training</c>"
+    },
+    {
+      "start": 75.71,
+      "duration": 0.0,
+      "text": "is the training loss and the training"
+    },
+    {
+      "start": 75.72,
+      "duration": 0.0,
+      "text": "is the training loss and the training algorithm<00:01:16.720><c> um</c><00:01:17.080><c> so</c><00:01:17.360><c> how</c><00:01:17.560><c> you</c><00:01:17.799><c> actually</c><00:01:18.320><c> train</c>"
+    },
+    {
+      "start": 78.67,
+      "duration": 0.0,
+      "text": "algorithm um so how you actually train"
+    },
+    {
+      "start": 78.68,
+      "duration": 0.0,
+      "text": "algorithm um so how you actually train these<00:01:18.840><c> models</c><00:01:19.640><c> then</c><00:01:19.759><c> it's</c><00:01:20.040><c> data</c><00:01:20.680><c> so</c><00:01:21.560><c> uh</c><00:01:21.759><c> what</c>"
+    },
+    {
+      "start": 81.91,
+      "duration": 0.0,
+      "text": "these models then it's data so uh what"
+    },
+    {
+      "start": 81.92,
+      "duration": 0.0,
+      "text": "these models then it's data so uh what do<00:01:22.000><c> you</c><00:01:22.159><c> train</c><00:01:22.439><c> these</c><00:01:22.640><c> models</c><00:01:23.159><c> on</c><00:01:24.159><c> um</c><00:01:24.520><c> the</c>"
+    },
+    {
+      "start": 84.71,
+      "duration": 0.0,
+      "text": "do you train these models on um the"
+    },
+    {
+      "start": 84.72,
+      "duration": 0.0,
+      "text": "do you train these models on um the evaluation<00:01:25.520><c> which</c><00:01:25.640><c> is</c><00:01:25.799><c> how</c><00:01:25.960><c> do</c><00:01:26.040><c> you</c><00:01:26.200><c> know</c>"
+    },
+    {
+      "start": 86.429,
+      "duration": 0.0,
+      "text": "evaluation which is how do you know"
+    },
+    {
+      "start": 86.439,
+      "duration": 0.0,
+      "text": "evaluation which is how do you know whether<00:01:26.600><c> you're</c><00:01:26.880><c> actually</c><00:01:27.119><c> making</c><00:01:27.920><c> progress</c>"
+    },
+    {
+      "start": 88.59,
+      "duration": 0.0,
+      "text": "whether you're actually making progress"
+    },
+    {
+      "start": 88.6,
+      "duration": 0.0,
+      "text": "whether you're actually making progress towards<00:01:29.000><c> the</c><00:01:29.159><c> goal</c><00:01:29.720><c> of</c><00:01:29.960><c> of</c><00:01:30.400><c> uh</c><00:01:30.920><c> llms</c><00:01:31.920><c> and</c><00:01:32.079><c> then</c>"
+    },
+    {
+      "start": 92.23,
+      "duration": 0.0,
+      "text": "towards the goal of of uh llms and then"
+    },
+    {
+      "start": 92.24,
+      "duration": 0.0,
+      "text": "towards the goal of of uh llms and then the<00:01:32.439><c> system</c><00:01:32.799><c> component</c><00:01:33.280><c> so</c><00:01:33.479><c> that</c><00:01:33.560><c> is</c><00:01:33.759><c> like</c><00:01:34.079><c> how</c>"
+    },
+    {
+      "start": 94.23,
+      "duration": 0.0,
+      "text": "the system component so that is like how"
+    },
+    {
+      "start": 94.24,
+      "duration": 0.0,
+      "text": "the system component so that is like how do<00:01:34.360><c> you</c><00:01:34.720><c> actually</c><00:01:35.119><c> make</c><00:01:35.360><c> these</c><00:01:35.560><c> models</c><00:01:36.000><c> run</c><00:01:36.640><c> on</c>"
+    },
+    {
+      "start": 97.23,
+      "duration": 0.0,
+      "text": "do you actually make these models run on"
+    },
+    {
+      "start": 97.24,
+      "duration": 0.0,
+      "text": "do you actually make these models run on uh<00:01:37.439><c> Modern</c><00:01:37.880><c> Hardware</c><00:01:38.439><c> which</c><00:01:38.560><c> is</c><00:01:38.680><c> really</c>"
+    },
+    {
+      "start": 98.87,
+      "duration": 0.0,
+      "text": "uh Modern Hardware which is really"
+    },
+    {
+      "start": 98.88,
+      "duration": 0.0,
+      "text": "uh Modern Hardware which is really important<00:01:39.240><c> because</c><00:01:39.399><c> these</c><00:01:39.520><c> models</c><00:01:39.759><c> are</c>"
+    },
+    {
+      "start": 99.91,
+      "duration": 0.0,
+      "text": "important because these models are"
+    },
+    {
+      "start": 99.92,
+      "duration": 0.0,
+      "text": "important because these models are really<00:01:40.159><c> large</c><00:01:40.960><c> um</c><00:01:41.159><c> so</c><00:01:41.759><c> now</c><00:01:42.000><c> more</c><00:01:42.200><c> than</c><00:01:42.399><c> ever</c>"
+    },
+    {
+      "start": 102.789,
+      "duration": 0.0,
+      "text": "really large um so now more than ever"
+    },
+    {
+      "start": 102.799,
+      "duration": 0.0,
+      "text": "really large um so now more than ever system<00:01:43.119><c> is</c><00:01:43.320><c> actually</c><00:01:43.560><c> really</c><00:01:43.799><c> an</c><00:01:44.000><c> important</c>"
+    },
+    {
+      "start": 104.389,
+      "duration": 0.0,
+      "text": "system is actually really an important"
+    },
+    {
+      "start": 104.399,
+      "duration": 0.0,
+      "text": "system is actually really an important topic<00:01:45.280><c> um</c><00:01:45.680><c> for</c>"
+    },
+    {
+      "start": 106.469,
+      "duration": 0.0,
+      "text": "topic um for"
+    },
+    {
+      "start": 106.479,
+      "duration": 0.0,
+      "text": "topic um for llms<00:01:47.479><c> so</c><00:01:48.079><c> those</c><00:01:48.360><c> five</c><00:01:48.600><c> components</c><00:01:49.520><c> um</c><00:01:49.880><c> You</c>"
+    },
+    {
+      "start": 110.109,
+      "duration": 0.0,
+      "text": "llms so those five components um You"
+    },
+    {
+      "start": 110.119,
+      "duration": 0.0,
+      "text": "llms so those five components um You probably<00:01:50.439><c> all</c><00:01:50.640><c> know</c><00:01:50.960><c> that</c><00:01:51.360><c> llms</c><00:01:52.119><c> and</c><00:01:52.240><c> if</c><00:01:52.360><c> you</c>"
+    },
+    {
+      "start": 112.429,
+      "duration": 0.0,
+      "text": "probably all know that llms and if you"
+    },
+    {
+      "start": 112.439,
+      "duration": 0.0,
+      "text": "probably all know that llms and if you don't<00:01:52.640><c> know</c><00:01:53.000><c> LMS</c><00:01:53.439><c> are</c><00:01:53.560><c> all</c><00:01:53.759><c> based</c><00:01:54.040><c> on</c>"
+    },
+    {
+      "start": 114.149,
+      "duration": 0.0,
+      "text": "don't know LMS are all based on"
+    },
+    {
+      "start": 114.159,
+      "duration": 0.0,
+      "text": "don't know LMS are all based on Transformers<00:01:54.840><c> or</c><00:01:55.000><c> at</c><00:01:55.119><c> least</c><00:01:55.360><c> some</c><00:01:55.600><c> version</c><00:01:55.880><c> of</c>"
+    },
+    {
+      "start": 116.389,
+      "duration": 0.0,
+      "text": "Transformers or at least some version of"
+    },
+    {
+      "start": 116.399,
+      "duration": 0.0,
+      "text": "Transformers or at least some version of Transformers<00:01:57.399><c> uh</c><00:01:57.520><c> I'm</c><00:01:57.719><c> actually</c><00:01:58.000><c> not</c><00:01:58.159><c> going</c>"
+    },
+    {
+      "start": 118.27,
+      "duration": 0.0,
+      "text": "Transformers uh I'm actually not going"
+    },
+    {
+      "start": 118.28,
+      "duration": 0.0,
+      "text": "Transformers uh I'm actually not going to<00:01:58.680><c> talk</c><00:01:59.119><c> about</c><00:01:59.439><c> the</c><00:01:59.520><c> AR</c><00:01:59.920><c> lecture</c><00:02:00.200><c> today</c><00:02:00.840><c> uh</c>"
+    },
+    {
+      "start": 120.99,
+      "duration": 0.0,
+      "text": "to talk about the AR lecture today uh"
+    },
+    {
+      "start": 121.0,
+      "duration": 0.0,
+      "text": "to talk about the AR lecture today uh one<00:02:01.280><c> because</c><00:02:01.479><c> I</c><00:02:01.600><c> gave</c><00:02:01.719><c> a</c><00:02:01.880><c> SE</c><00:02:02.240><c> lecture</c><00:02:02.960><c> on</c><00:02:03.719><c> um</c>"
+    },
+    {
+      "start": 124.149,
+      "duration": 0.0,
+      "text": "one because I gave a SE lecture on um"
+    },
+    {
+      "start": 124.159,
+      "duration": 0.0,
+      "text": "one because I gave a SE lecture on um Transformers<00:02:05.159><c> a</c><00:02:05.280><c> few</c><00:02:05.479><c> weeks</c><00:02:05.719><c> ago</c><00:02:06.560><c> and</c><00:02:06.719><c> two</c>"
+    },
+    {
+      "start": 126.95,
+      "duration": 0.0,
+      "text": "Transformers a few weeks ago and two"
+    },
+    {
+      "start": 126.96,
+      "duration": 0.0,
+      "text": "Transformers a few weeks ago and two because<00:02:07.159><c> you</c><00:02:07.240><c> can</c><00:02:07.399><c> find</c><00:02:07.560><c> so</c><00:02:07.880><c> much</c><00:02:08.080><c> information</c>"
+    },
+    {
+      "start": 128.469,
+      "duration": 0.0,
+      "text": "because you can find so much information"
+    },
+    {
+      "start": 128.479,
+      "duration": 0.0,
+      "text": "because you can find so much information online<00:02:09.239><c> on</c><00:02:09.720><c> uh</c><00:02:09.879><c> Transformers</c><00:02:10.599><c> but</c><00:02:10.679><c> I</c><00:02:10.800><c> think</c>"
+    },
+    {
+      "start": 130.91,
+      "duration": 0.0,
+      "text": "online on uh Transformers but I think"
+    },
+    {
+      "start": 130.92,
+      "duration": 0.0,
+      "text": "online on uh Transformers but I think you<00:02:11.039><c> can</c><00:02:11.440><c> it's</c><00:02:12.239><c> there's</c><00:02:12.440><c> much</c><00:02:12.680><c> less</c>"
+    },
+    {
+      "start": 132.91,
+      "duration": 0.0,
+      "text": "you can it's there's much less"
+    },
+    {
+      "start": 132.92,
+      "duration": 0.0,
+      "text": "you can it's there's much less information<00:02:13.319><c> about</c><00:02:13.560><c> the</c><00:02:13.680><c> other</c><00:02:13.879><c> four</c><00:02:14.200><c> topics</c>"
+    },
+    {
+      "start": 134.47,
+      "duration": 0.0,
+      "text": "information about the other four topics"
+    },
+    {
+      "start": 134.48,
+      "duration": 0.0,
+      "text": "information about the other four topics so<00:02:14.599><c> I</c><00:02:14.760><c> really</c><00:02:14.879><c> want</c><00:02:15.000><c> to</c><00:02:15.519><c> talk</c><00:02:15.760><c> about</c><00:02:16.480><c> those</c><00:02:17.480><c> um</c>"
+    },
+    {
+      "start": 137.83,
+      "duration": 0.0,
+      "text": "so I really want to talk about those um"
+    },
+    {
+      "start": 137.84,
+      "duration": 0.0,
+      "text": "so I really want to talk about those um another<00:02:18.120><c> thing</c><00:02:18.280><c> to</c><00:02:18.440><c> say</c><00:02:18.720><c> is</c><00:02:18.840><c> that</c><00:02:19.000><c> most</c><00:02:19.200><c> of</c>"
+    },
+    {
+      "start": 139.43,
+      "duration": 0.0,
+      "text": "another thing to say is that most of"
+    },
+    {
+      "start": 139.44,
+      "duration": 0.0,
+      "text": "another thing to say is that most of Academia<00:02:20.319><c> actually</c><00:02:20.640><c> focuses</c><00:02:21.080><c> on</c>"
+    },
+    {
+      "start": 141.229,
+      "duration": 0.0,
+      "text": "Academia actually focuses on"
+    },
+    {
+      "start": 141.239,
+      "duration": 0.0,
+      "text": "Academia actually focuses on architecture<00:02:22.239><c> and</c><00:02:22.440><c> training</c><00:02:22.879><c> algorithm</c><00:02:23.319><c> and</c>"
+    },
+    {
+      "start": 143.509,
+      "duration": 0.0,
+      "text": "architecture and training algorithm and"
+    },
+    {
+      "start": 143.519,
+      "duration": 0.0,
+      "text": "architecture and training algorithm and losses<00:02:24.519><c> um</c><00:02:25.040><c> as</c><00:02:25.239><c> academics</c><00:02:25.720><c> and</c><00:02:25.840><c> I've</c><00:02:26.000><c> done</c>"
+    },
+    {
+      "start": 146.19,
+      "duration": 0.0,
+      "text": "losses um as academics and I've done"
+    },
+    {
+      "start": 146.2,
+      "duration": 0.0,
+      "text": "losses um as academics and I've done that<00:02:26.360><c> for</c><00:02:26.519><c> a</c><00:02:26.680><c> lot</c><00:02:27.440><c> big</c><00:02:27.680><c> part</c><00:02:27.879><c> of</c><00:02:28.000><c> my</c><00:02:28.200><c> career</c><00:02:28.800><c> is</c>"
+    },
+    {
+      "start": 148.99,
+      "duration": 0.0,
+      "text": "that for a lot big part of my career is"
+    },
+    {
+      "start": 149.0,
+      "duration": 0.0,
+      "text": "that for a lot big part of my career is simply<00:02:30.040><c> we</c><00:02:30.160><c> like</c><00:02:30.480><c> thinking</c><00:02:30.879><c> that</c><00:02:31.080><c> this</c><00:02:31.200><c> is</c><00:02:31.560><c> uh</c>"
+    },
+    {
+      "start": 151.949,
+      "duration": 0.0,
+      "text": "simply we like thinking that this is uh"
+    },
+    {
+      "start": 151.959,
+      "duration": 0.0,
+      "text": "simply we like thinking that this is uh like<00:02:32.080><c> we</c><00:02:32.360><c> make</c><00:02:32.599><c> new</c><00:02:32.959><c> architectures</c><00:02:33.599><c> new</c>"
+    },
+    {
+      "start": 153.79,
+      "duration": 0.0,
+      "text": "like we make new architectures new"
+    },
+    {
+      "start": 153.8,
+      "duration": 0.0,
+      "text": "like we make new architectures new models<00:02:34.319><c> and</c><00:02:34.599><c> it</c><00:02:34.840><c> it</c><00:02:35.160><c> seems</c><00:02:35.599><c> like</c><00:02:35.760><c> it's</c><00:02:35.959><c> very</c>"
+    },
+    {
+      "start": 156.19,
+      "duration": 0.0,
+      "text": "models and it it seems like it's very"
+    },
+    {
+      "start": 156.2,
+      "duration": 0.0,
+      "text": "models and it it seems like it's very important<00:02:37.000><c> but</c><00:02:37.160><c> in</c><00:02:37.360><c> reality</c><00:02:37.959><c> honestly</c><00:02:38.319><c> what</c>"
+    },
+    {
+      "start": 158.47,
+      "duration": 0.0,
+      "text": "important but in reality honestly what"
+    },
+    {
+      "start": 158.48,
+      "duration": 0.0,
+      "text": "important but in reality honestly what matters<00:02:38.760><c> in</c><00:02:38.920><c> practice</c><00:02:39.400><c> is</c><00:02:39.560><c> mostly</c><00:02:39.920><c> the</c><00:02:40.080><c> three</c>"
+    },
+    {
+      "start": 160.71,
+      "duration": 0.0,
+      "text": "matters in practice is mostly the three"
+    },
+    {
+      "start": 160.72,
+      "duration": 0.0,
+      "text": "matters in practice is mostly the three other<00:02:41.120><c> topics</c><00:02:41.560><c> so</c><00:02:41.800><c> data</c><00:02:42.640><c> evaluation</c><00:02:43.159><c> and</c>"
+    },
+    {
+      "start": 163.35,
+      "duration": 0.0,
+      "text": "other topics so data evaluation and"
+    },
+    {
+      "start": 163.36,
+      "duration": 0.0,
+      "text": "other topics so data evaluation and systems<00:02:44.280><c> uh</c><00:02:44.440><c> which</c><00:02:44.560><c> is</c><00:02:44.720><c> what</c><00:02:44.920><c> of</c><00:02:45.280><c> most</c><00:02:45.480><c> of</c>"
+    },
+    {
+      "start": 165.71,
+      "duration": 0.0,
+      "text": "systems uh which is what of most of"
+    },
+    {
+      "start": 165.72,
+      "duration": 0.0,
+      "text": "systems uh which is what of most of Industry<00:02:46.400><c> actually</c><00:02:46.680><c> focuses</c><00:02:47.159><c> on</c><00:02:48.040><c> um</c><00:02:48.480><c> so</c>"
+    },
+    {
+      "start": 168.67,
+      "duration": 0.0,
+      "text": "Industry actually focuses on um so"
+    },
+    {
+      "start": 168.68,
+      "duration": 0.0,
+      "text": "Industry actually focuses on um so that's<00:02:48.840><c> also</c><00:02:49.040><c> one</c><00:02:49.159><c> of</c><00:02:49.280><c> the</c><00:02:49.360><c> reason</c><00:02:49.640><c> why</c><00:02:49.760><c> I</c>"
+    },
+    {
+      "start": 169.83,
+      "duration": 0.0,
+      "text": "that's also one of the reason why I"
+    },
+    {
+      "start": 169.84,
+      "duration": 0.0,
+      "text": "that's also one of the reason why I don't<00:02:49.959><c> want</c><00:02:50.080><c> to</c><00:02:50.200><c> talk</c><00:02:50.360><c> too</c><00:02:50.519><c> much</c><00:02:50.680><c> about</c><00:02:50.879><c> the</c>"
+    },
+    {
+      "start": 170.99,
+      "duration": 0.0,
+      "text": "don't want to talk too much about the"
+    },
+    {
+      "start": 171.0,
+      "duration": 0.0,
+      "text": "don't want to talk too much about the architecture<00:02:51.920><c> uh</c><00:02:52.080><c> because</c><00:02:52.239><c> really</c><00:02:52.440><c> the</c><00:02:52.560><c> rest</c>"
+    },
+    {
+      "start": 172.75,
+      "duration": 0.0,
+      "text": "architecture uh because really the rest"
+    },
+    {
+      "start": 172.76,
+      "duration": 0.0,
+      "text": "architecture uh because really the rest is<00:02:52.920><c> super</c>"
+    },
+    {
+      "start": 173.869,
+      "duration": 0.0,
+      "text": "is super"
+    },
+    {
+      "start": 173.879,
+      "duration": 0.0,
+      "text": "is super important<00:02:54.879><c> um</c><00:02:55.239><c> great</c><00:02:55.519><c> so</c><00:02:55.760><c> overview</c><00:02:56.159><c> of</c><00:02:56.280><c> the</c>"
+    },
+    {
+      "start": 176.43,
+      "duration": 0.0,
+      "text": "important um great so overview of the"
+    },
+    {
+      "start": 176.44,
+      "duration": 0.0,
+      "text": "important um great so overview of the lecture<00:02:57.200><c> I'll</c><00:02:57.360><c> be</c><00:02:57.519><c> talking</c><00:02:57.760><c> about</c>"
+    },
+    {
+      "start": 177.949,
+      "duration": 0.0,
+      "text": "lecture I'll be talking about"
+    },
+    {
+      "start": 177.959,
+      "duration": 0.0,
+      "text": "lecture I'll be talking about pre-training<00:02:58.560><c> so</c><00:02:58.760><c> pre-training</c><00:02:59.480><c> uh</c><00:02:59.560><c> you</c>"
+    },
+    {
+      "start": 179.869,
+      "duration": 0.0,
+      "text": "pre-training so pre-training uh you"
+    },
+    {
+      "start": 179.879,
+      "duration": 0.0,
+      "text": "pre-training so pre-training uh you probably<00:03:00.080><c> heard</c><00:03:00.319><c> that</c><00:03:00.480><c> word</c><00:03:00.800><c> this</c><00:03:00.879><c> is</c><00:03:01.080><c> the</c>"
+    },
+    {
+      "start": 181.35,
+      "duration": 0.0,
+      "text": "probably heard that word this is the"
+    },
+    {
+      "start": 181.36,
+      "duration": 0.0,
+      "text": "probably heard that word this is the general<00:03:01.760><c> word</c><00:03:02.200><c> this</c><00:03:02.319><c> is</c><00:03:02.480><c> kind</c><00:03:02.640><c> of</c><00:03:02.760><c> the</c>"
+    },
+    {
+      "start": 182.949,
+      "duration": 0.0,
+      "text": "general word this is kind of the"
+    },
+    {
+      "start": 182.959,
+      "duration": 0.0,
+      "text": "general word this is kind of the classical<00:03:03.720><c> language</c><00:03:04.120><c> modeling</c><00:03:05.120><c> uh</c><00:03:05.280><c> Paradigm</c>"
+    },
+    {
+      "start": 186.11,
+      "duration": 0.0,
+      "text": "classical language modeling uh Paradigm"
+    },
+    {
+      "start": 186.12,
+      "duration": 0.0,
+      "text": "classical language modeling uh Paradigm uh<00:03:06.239><c> where</c><00:03:06.360><c> you</c><00:03:06.519><c> basically</c><00:03:06.799><c> train</c><00:03:07.040><c> your</c>"
+    },
+    {
+      "start": 187.19,
+      "duration": 0.0,
+      "text": "uh where you basically train your"
+    },
+    {
+      "start": 187.2,
+      "duration": 0.0,
+      "text": "uh where you basically train your language<00:03:07.480><c> model</c><00:03:07.799><c> to</c><00:03:08.440><c> essentially</c><00:03:08.879><c> model</c><00:03:09.200><c> all</c>"
+    },
+    {
+      "start": 189.35,
+      "duration": 0.0,
+      "text": "language model to essentially model all"
+    },
+    {
+      "start": 189.36,
+      "duration": 0.0,
+      "text": "language model to essentially model all of<00:03:09.720><c> internet</c><00:03:10.720><c> and</c><00:03:10.879><c> then</c><00:03:11.000><c> there's</c><00:03:11.159><c> a</c><00:03:11.319><c> post</c>"
+    },
+    {
+      "start": 191.55,
+      "duration": 0.0,
+      "text": "of internet and then there's a post"
+    },
+    {
+      "start": 191.56,
+      "duration": 0.0,
+      "text": "of internet and then there's a post training<00:03:12.040><c> which</c><00:03:12.120><c> is</c><00:03:12.239><c> a</c><00:03:12.360><c> more</c><00:03:12.560><c> recent</c><00:03:12.879><c> Paradigm</c>"
+    },
+    {
+      "start": 193.35,
+      "duration": 0.0,
+      "text": "training which is a more recent Paradigm"
+    },
+    {
+      "start": 193.36,
+      "duration": 0.0,
+      "text": "training which is a more recent Paradigm which<00:03:13.480><c> is</c><00:03:13.640><c> taking</c><00:03:13.959><c> these</c><00:03:14.120><c> large</c><00:03:14.400><c> language</c>"
+    },
+    {
+      "start": 194.75,
+      "duration": 0.0,
+      "text": "which is taking these large language"
+    },
+    {
+      "start": 194.76,
+      "duration": 0.0,
+      "text": "which is taking these large language models<00:03:15.360><c> and</c><00:03:15.519><c> making</c><00:03:15.920><c> them</c><00:03:16.120><c> essentially</c><00:03:16.599><c> AI</c>"
+    },
+    {
+      "start": 196.99,
+      "duration": 0.0,
+      "text": "models and making them essentially AI"
+    },
+    {
+      "start": 197.0,
+      "duration": 0.0,
+      "text": "models and making them essentially AI assistants<00:03:18.000><c> um</c><00:03:18.159><c> so</c><00:03:18.440><c> this</c><00:03:18.560><c> is</c><00:03:18.959><c> more</c><00:03:19.159><c> of</c><00:03:19.280><c> a</c>"
+    },
+    {
+      "start": 199.509,
+      "duration": 0.0,
+      "text": "assistants um so this is more of a"
+    },
+    {
+      "start": 199.519,
+      "duration": 0.0,
+      "text": "assistants um so this is more of a recent<00:03:19.879><c> Trend</c><00:03:20.400><c> since</c><00:03:20.640><c> Chad</c><00:03:21.319><c> GPT</c><00:03:22.319><c> uh</c><00:03:22.480><c> so</c><00:03:22.799><c> if</c><00:03:22.920><c> you</c>"
+    },
+    {
+      "start": 203.07,
+      "duration": 0.0,
+      "text": "recent Trend since Chad GPT uh so if you"
+    },
+    {
+      "start": 203.08,
+      "duration": 0.0,
+      "text": "recent Trend since Chad GPT uh so if you ever<00:03:23.280><c> heard</c><00:03:23.440><c> of</c><00:03:23.560><c> gpt3</c><00:03:24.080><c> or</c><00:03:24.239><c> gpt2</c><00:03:25.080><c> that's</c><00:03:25.280><c> really</c>"
+    },
+    {
+      "start": 205.55,
+      "duration": 0.0,
+      "text": "ever heard of gpt3 or gpt2 that's really"
+    },
+    {
+      "start": 205.56,
+      "duration": 0.0,
+      "text": "ever heard of gpt3 or gpt2 that's really pre-training<00:03:26.280><c> land</c><00:03:27.280><c> uh</c><00:03:27.519><c> if</c><00:03:27.640><c> you</c><00:03:27.840><c> heard</c><00:03:28.040><c> of</c>"
+    },
+    {
+      "start": 208.149,
+      "duration": 0.0,
+      "text": "pre-training land uh if you heard of"
+    },
+    {
+      "start": 208.159,
+      "duration": 0.0,
+      "text": "pre-training land uh if you heard of chat<00:03:28.400><c> GPT</c><00:03:28.720><c> which</c><00:03:28.840><c> you</c><00:03:28.959><c> probably</c><00:03:29.319><c> have</c><00:03:29.840><c> this</c><00:03:29.959><c> is</c>"
+    },
+    {
+      "start": 210.07,
+      "duration": 0.0,
+      "text": "chat GPT which you probably have this is"
+    },
+    {
+      "start": 210.08,
+      "duration": 0.0,
+      "text": "chat GPT which you probably have this is really<00:03:30.360><c> posttraining</c><00:03:30.799><c> land</c><00:03:31.799><c> uh</c><00:03:31.879><c> so</c><00:03:32.040><c> I'll</c><00:03:32.159><c> be</c>"
+    },
+    {
+      "start": 212.309,
+      "duration": 0.0,
+      "text": "really posttraining land uh so I'll be"
+    },
+    {
+      "start": 212.319,
+      "duration": 0.0,
+      "text": "really posttraining land uh so I'll be talking<00:03:32.599><c> about</c><00:03:32.920><c> both</c><00:03:33.280><c> but</c><00:03:33.439><c> I'll</c><00:03:33.560><c> start</c><00:03:33.799><c> with</c>"
+    },
+    {
+      "start": 214.19,
+      "duration": 0.0,
+      "text": "talking about both but I'll start with"
+    },
+    {
+      "start": 214.2,
+      "duration": 0.0,
+      "text": "talking about both but I'll start with pre-training<00:03:35.200><c> and</c><00:03:35.439><c> uh</c><00:03:35.640><c> specifically</c><00:03:36.120><c> I'll</c>"
+    },
+    {
+      "start": 216.67,
+      "duration": 0.0,
+      "text": "pre-training and uh specifically I'll"
+    },
+    {
+      "start": 216.68,
+      "duration": 0.0,
+      "text": "pre-training and uh specifically I'll talk<00:03:36.879><c> about</c><00:03:37.200><c> what</c><00:03:37.360><c> is</c><00:03:37.480><c> the</c><00:03:37.879><c> task</c><00:03:38.360><c> of</c>"
+    },
+    {
+      "start": 218.509,
+      "duration": 0.0,
+      "text": "talk about what is the task of"
+    },
+    {
+      "start": 218.519,
+      "duration": 0.0,
+      "text": "talk about what is the task of pre-training<00:03:39.040><c> llms</c><00:03:39.840><c> and</c><00:03:40.000><c> what</c><00:03:40.080><c> is</c><00:03:40.200><c> the</c><00:03:40.319><c> laws</c>"
+    },
+    {
+      "start": 220.55,
+      "duration": 0.0,
+      "text": "pre-training llms and what is the laws"
+    },
+    {
+      "start": 220.56,
+      "duration": 0.0,
+      "text": "pre-training llms and what is the laws that<00:03:40.720><c> people</c><00:03:41.120><c> actually</c>"
+    },
+    {
+      "start": 222.27,
+      "duration": 0.0,
+      "text": "that people actually"
+    },
+    {
+      "start": 222.28,
+      "duration": 0.0,
+      "text": "that people actually use<00:03:43.280><c> so</c><00:03:43.519><c> language</c><00:03:43.879><c> modeling</c><00:03:44.480><c> this</c><00:03:44.560><c> is</c><00:03:44.720><c> a</c><00:03:45.360><c> quick</c>"
+    },
+    {
+      "start": 225.949,
+      "duration": 0.0,
+      "text": "use so language modeling this is a quick"
+    },
+    {
+      "start": 225.959,
+      "duration": 0.0,
+      "text": "use so language modeling this is a quick recap<00:03:46.959><c> uh</c><00:03:47.159><c> language</c><00:03:47.480><c> models</c><00:03:47.799><c> at</c><00:03:47.920><c> a</c><00:03:48.040><c> high</c><00:03:48.280><c> level</c>"
+    },
+    {
+      "start": 228.67,
+      "duration": 0.0,
+      "text": "recap uh language models at a high level"
+    },
+    {
+      "start": 228.68,
+      "duration": 0.0,
+      "text": "recap uh language models at a high level are<00:03:48.879><c> simply</c><00:03:49.319><c> models</c><00:03:49.840><c> of</c><00:03:50.080><c> probability</c>"
+    },
+    {
+      "start": 230.63,
+      "duration": 0.0,
+      "text": "are simply models of probability"
+    },
+    {
+      "start": 230.64,
+      "duration": 0.0,
+      "text": "are simply models of probability distribution<00:03:51.159><c> over</c><00:03:51.439><c> sequences</c><00:03:52.239><c> of</c><00:03:52.400><c> tokens</c><00:03:52.799><c> or</c>"
+    },
+    {
+      "start": 232.949,
+      "duration": 0.0,
+      "text": "distribution over sequences of tokens or"
+    },
+    {
+      "start": 232.959,
+      "duration": 0.0,
+      "text": "distribution over sequences of tokens or of<00:03:53.120><c> words</c><00:03:53.599><c> so</c><00:03:53.799><c> it's</c><00:03:54.040><c> basically</c><00:03:54.519><c> some</c><00:03:55.360><c> uh</c><00:03:55.560><c> model</c>"
+    },
+    {
+      "start": 236.03,
+      "duration": 0.0,
+      "text": "of words so it's basically some uh model"
+    },
+    {
+      "start": 236.04,
+      "duration": 0.0,
+      "text": "of words so it's basically some uh model of<00:03:56.239><c> P</c><00:03:56.480><c> of</c><00:03:56.720><c> X1</c><00:03:57.200><c> to</c><00:03:57.480><c> XL</c><00:03:58.000><c> where</c><00:03:58.200><c> X1</c><00:03:58.680><c> is</c><00:03:58.879><c> basically</c>"
+    },
+    {
+      "start": 239.309,
+      "duration": 0.0,
+      "text": "of P of X1 to XL where X1 is basically"
+    },
+    {
+      "start": 239.319,
+      "duration": 0.0,
+      "text": "of P of X1 to XL where X1 is basically word<00:03:59.560><c> one</c><00:03:59.879><c> and</c><00:04:00.040><c> Excel</c><00:04:00.360><c> is</c><00:04:00.519><c> the</c><00:04:00.680><c> last</c><00:04:00.959><c> one</c><00:04:01.560><c> in</c>"
+    },
+    {
+      "start": 241.67,
+      "duration": 0.0,
+      "text": "word one and Excel is the last one in"
+    },
+    {
+      "start": 241.68,
+      "duration": 0.0,
+      "text": "word one and Excel is the last one in the<00:04:01.840><c> sequence</c><00:04:02.280><c> or</c><00:04:02.400><c> in</c><00:04:02.519><c> the</c><00:04:03.079><c> sentence</c><00:04:04.079><c> um</c><00:04:04.280><c> so</c>"
+    },
+    {
+      "start": 244.47,
+      "duration": 0.0,
+      "text": "the sequence or in the sentence um so"
+    },
+    {
+      "start": 244.48,
+      "duration": 0.0,
+      "text": "the sequence or in the sentence um so very<00:04:04.680><c> concretely</c><00:04:05.319><c> if</c><00:04:05.400><c> you</c><00:04:05.519><c> have</c><00:04:05.640><c> a</c><00:04:05.799><c> sentence</c>"
+    },
+    {
+      "start": 246.19,
+      "duration": 0.0,
+      "text": "very concretely if you have a sentence"
+    },
+    {
+      "start": 246.2,
+      "duration": 0.0,
+      "text": "very concretely if you have a sentence like<00:04:06.480><c> the</c><00:04:06.760><c> mouse</c><00:04:07.120><c> ate</c><00:04:07.400><c> the</c><00:04:07.560><c> cheese</c><00:04:08.319><c> what</c><00:04:08.480><c> the</c>"
+    },
+    {
+      "start": 248.589,
+      "duration": 0.0,
+      "text": "like the mouse ate the cheese what the"
+    },
+    {
+      "start": 248.599,
+      "duration": 0.0,
+      "text": "like the mouse ate the cheese what the language<00:04:08.920><c> model</c><00:04:09.319><c> gives</c><00:04:09.560><c> you</c><00:04:09.959><c> is</c><00:04:10.120><c> simply</c><00:04:10.720><c> a</c>"
+    },
+    {
+      "start": 250.949,
+      "duration": 0.0,
+      "text": "language model gives you is simply a"
+    },
+    {
+      "start": 250.959,
+      "duration": 0.0,
+      "text": "language model gives you is simply a probability<00:04:11.959><c> of</c><00:04:12.360><c> this</c><00:04:12.760><c> sentence</c><00:04:13.200><c> being</c>"
+    },
+    {
+      "start": 253.429,
+      "duration": 0.0,
+      "text": "probability of this sentence being"
+    },
+    {
+      "start": 253.439,
+      "duration": 0.0,
+      "text": "probability of this sentence being uttered<00:04:13.840><c> by</c><00:04:13.959><c> a</c><00:04:14.079><c> human</c><00:04:14.360><c> or</c><00:04:14.560><c> being</c><00:04:14.760><c> found</c><00:04:15.120><c> on</c><00:04:15.439><c> on</c>"
+    },
+    {
+      "start": 256.069,
+      "duration": 0.0,
+      "text": "uttered by a human or being found on on"
+    },
+    {
+      "start": 256.079,
+      "duration": 0.0,
+      "text": "uttered by a human or being found on on online<00:04:17.079><c> uh</c><00:04:17.160><c> so</c><00:04:17.320><c> if</c><00:04:17.400><c> you</c><00:04:17.519><c> have</c><00:04:17.680><c> another</c>"
+    },
+    {
+      "start": 257.99,
+      "duration": 0.0,
+      "text": "online uh so if you have another"
+    },
+    {
+      "start": 258.0,
+      "duration": 0.0,
+      "text": "online uh so if you have another sentence<00:04:18.600><c> like</c><00:04:18.880><c> the</c><00:04:19.199><c> the</c><00:04:19.400><c> mouse</c><00:04:20.199><c> at</c><00:04:20.720><c> cheese</c><00:04:21.720><c> uh</c>"
+    },
+    {
+      "start": 261.99,
+      "duration": 0.0,
+      "text": "sentence like the the mouse at cheese uh"
+    },
+    {
+      "start": 262.0,
+      "duration": 0.0,
+      "text": "sentence like the the mouse at cheese uh here<00:04:22.199><c> there's</c><00:04:22.639><c> grammatical</c><00:04:23.160><c> mistakes</c><00:04:23.600><c> so</c><00:04:23.800><c> the</c>"
+    },
+    {
+      "start": 263.909,
+      "duration": 0.0,
+      "text": "here there's grammatical mistakes so the"
+    },
+    {
+      "start": 263.919,
+      "duration": 0.0,
+      "text": "here there's grammatical mistakes so the model<00:04:24.160><c> should</c><00:04:24.360><c> know</c><00:04:24.600><c> that</c><00:04:24.759><c> this</c><00:04:25.320><c> uh</c><00:04:25.520><c> should</c>"
+    },
+    {
+      "start": 265.749,
+      "duration": 0.0,
+      "text": "model should know that this uh should"
+    },
+    {
+      "start": 265.759,
+      "duration": 0.0,
+      "text": "model should know that this uh should have<00:04:25.880><c> some</c><00:04:26.120><c> syntactic</c><00:04:26.840><c> knowledge</c><00:04:27.400><c> so</c><00:04:27.520><c> it</c>"
+    },
+    {
+      "start": 267.629,
+      "duration": 0.0,
+      "text": "have some syntactic knowledge so it"
+    },
+    {
+      "start": 267.639,
+      "duration": 0.0,
+      "text": "have some syntactic knowledge so it should<00:04:27.800><c> know</c><00:04:27.960><c> that</c><00:04:28.199><c> this</c><00:04:28.520><c> has</c><00:04:28.800><c> less</c>"
+    },
+    {
+      "start": 269.27,
+      "duration": 0.0,
+      "text": "should know that this has less"
+    },
+    {
+      "start": 269.28,
+      "duration": 0.0,
+      "text": "should know that this has less likelihood<00:04:30.160><c> of</c><00:04:30.360><c> appearing</c>"
+    },
+    {
+      "start": 271.469,
+      "duration": 0.0,
+      "text": "likelihood of appearing"
+    },
+    {
+      "start": 271.479,
+      "duration": 0.0,
+      "text": "likelihood of appearing online<00:04:32.479><c> uh</c><00:04:32.639><c> if</c><00:04:32.720><c> you</c><00:04:32.880><c> have</c><00:04:33.080><c> another</c><00:04:33.720><c> sentence</c>"
+    },
+    {
+      "start": 274.189,
+      "duration": 0.0,
+      "text": "online uh if you have another sentence"
+    },
+    {
+      "start": 274.199,
+      "duration": 0.0,
+      "text": "online uh if you have another sentence like<00:04:34.440><c> the</c><00:04:34.639><c> cheese</c><00:04:35.080><c> ate</c><00:04:35.400><c> the</c><00:04:35.560><c> mouse</c><00:04:36.440><c> uh</c><00:04:36.560><c> then</c>"
+    },
+    {
+      "start": 276.749,
+      "duration": 0.0,
+      "text": "like the cheese ate the mouse uh then"
+    },
+    {
+      "start": 276.759,
+      "duration": 0.0,
+      "text": "like the cheese ate the mouse uh then the<00:04:36.880><c> model</c><00:04:37.199><c> should</c><00:04:37.400><c> hopefully</c><00:04:37.840><c> know</c><00:04:38.520><c> about</c>"
+    },
+    {
+      "start": 278.95,
+      "duration": 0.0,
+      "text": "the model should hopefully know about"
+    },
+    {
+      "start": 278.96,
+      "duration": 0.0,
+      "text": "the model should hopefully know about the<00:04:39.120><c> fact</c><00:04:39.440><c> that</c><00:04:39.800><c> usually</c><00:04:40.240><c> cheese</c><00:04:40.600><c> don't</c><00:04:40.800><c> eat</c>"
+    },
+    {
+      "start": 280.99,
+      "duration": 0.0,
+      "text": "the fact that usually cheese don't eat"
+    },
+    {
+      "start": 281.0,
+      "duration": 0.0,
+      "text": "the fact that usually cheese don't eat Mouse<00:04:41.880><c> um</c><00:04:42.120><c> so</c><00:04:42.360><c> there's</c><00:04:42.560><c> some</c><00:04:42.759><c> semantic</c>"
+    },
+    {
+      "start": 283.15,
+      "duration": 0.0,
+      "text": "Mouse um so there's some semantic"
+    },
+    {
+      "start": 283.16,
+      "duration": 0.0,
+      "text": "Mouse um so there's some semantic knowledge<00:04:43.520><c> and</c><00:04:43.680><c> this</c><00:04:43.759><c> is</c><00:04:43.960><c> less</c><00:04:44.160><c> likely</c><00:04:44.400><c> than</c>"
+    },
+    {
+      "start": 284.51,
+      "duration": 0.0,
+      "text": "knowledge and this is less likely than"
+    },
+    {
+      "start": 284.52,
+      "duration": 0.0,
+      "text": "knowledge and this is less likely than the<00:04:44.639><c> first</c><00:04:44.880><c> sentence</c><00:04:45.320><c> so</c><00:04:45.479><c> this</c><00:04:45.560><c> is</c><00:04:45.759><c> basically</c>"
+    },
+    {
+      "start": 286.029,
+      "duration": 0.0,
+      "text": "the first sentence so this is basically"
+    },
+    {
+      "start": 286.039,
+      "duration": 0.0,
+      "text": "the first sentence so this is basically at<00:04:46.160><c> a</c><00:04:46.240><c> high</c><00:04:46.400><c> level</c><00:04:46.639><c> what</c><00:04:46.840><c> language</c><00:04:47.199><c> models</c><00:04:47.960><c> are</c>"
+    },
+    {
+      "start": 289.23,
+      "duration": 0.0,
+      "text": "at a high level what language models are"
+    },
+    {
+      "start": 289.24,
+      "duration": 0.0,
+      "text": "at a high level what language models are um<00:04:50.240><c> one</c><00:04:50.440><c> word</c><00:04:50.680><c> that</c><00:04:50.759><c> you</c><00:04:50.960><c> probably</c><00:04:51.280><c> have</c><00:04:51.440><c> been</c>"
+    },
+    {
+      "start": 291.59,
+      "duration": 0.0,
+      "text": "um one word that you probably have been"
+    },
+    {
+      "start": 291.6,
+      "duration": 0.0,
+      "text": "um one word that you probably have been hearing<00:04:51.840><c> a</c><00:04:51.919><c> lot</c><00:04:52.039><c> in</c><00:04:52.160><c> the</c><00:04:52.240><c> news</c><00:04:52.479><c> are</c><00:04:52.639><c> generative</c>"
+    },
+    {
+      "start": 293.07,
+      "duration": 0.0,
+      "text": "hearing a lot in the news are generative"
+    },
+    {
+      "start": 293.08,
+      "duration": 0.0,
+      "text": "hearing a lot in the news are generative models<00:04:54.000><c> uh</c><00:04:54.120><c> so</c><00:04:54.320><c> this</c><00:04:54.440><c> is</c><00:04:54.639><c> just</c><00:04:54.800><c> something</c><00:04:55.120><c> that</c>"
+    },
+    {
+      "start": 295.27,
+      "duration": 0.0,
+      "text": "models uh so this is just something that"
+    },
+    {
+      "start": 295.28,
+      "duration": 0.0,
+      "text": "models uh so this is just something that can<00:04:55.520><c> generate</c><00:04:56.240><c> models</c><00:04:56.600><c> that</c><00:04:56.720><c> can</c><00:04:56.840><c> generate</c>"
+    },
+    {
+      "start": 297.31,
+      "duration": 0.0,
+      "text": "can generate models that can generate"
+    },
+    {
+      "start": 297.32,
+      "duration": 0.0,
+      "text": "can generate models that can generate sentences<00:04:57.800><c> or</c><00:04:57.960><c> can</c><00:04:58.240><c> generate</c><00:04:58.600><c> some</c><00:04:58.840><c> data</c><00:04:59.479><c> uh</c>"
+    },
+    {
+      "start": 299.71,
+      "duration": 0.0,
+      "text": "sentences or can generate some data uh"
+    },
+    {
+      "start": 299.72,
+      "duration": 0.0,
+      "text": "sentences or can generate some data uh the<00:04:59.800><c> reason</c><00:05:00.039><c> why</c><00:05:00.160><c> we</c><00:05:00.240><c> say</c><00:05:00.400><c> language</c><00:05:00.720><c> models</c>"
+    },
+    {
+      "start": 300.99,
+      "duration": 0.0,
+      "text": "the reason why we say language models"
+    },
+    {
+      "start": 301.0,
+      "duration": 0.0,
+      "text": "the reason why we say language models are<00:05:01.160><c> generative</c><00:05:01.479><c> models</c><00:05:01.840><c> is</c><00:05:01.919><c> that</c><00:05:02.120><c> once</c><00:05:02.280><c> you</c>"
+    },
+    {
+      "start": 302.51,
+      "duration": 0.0,
+      "text": "are generative models is that once you"
+    },
+    {
+      "start": 302.52,
+      "duration": 0.0,
+      "text": "are generative models is that once you have<00:05:03.000><c> a</c><00:05:03.160><c> model</c><00:05:03.440><c> of</c><00:05:03.560><c> a</c><00:05:03.680><c> distribution</c><00:05:04.320><c> you</c><00:05:04.440><c> can</c>"
+    },
+    {
+      "start": 304.59,
+      "duration": 0.0,
+      "text": "have a model of a distribution you can"
+    },
+    {
+      "start": 304.6,
+      "duration": 0.0,
+      "text": "have a model of a distribution you can simply<00:05:04.919><c> sample</c><00:05:05.320><c> from</c><00:05:05.520><c> this</c><00:05:05.680><c> model</c><00:05:06.160><c> and</c><00:05:06.280><c> now</c><00:05:06.400><c> we</c>"
+    },
+    {
+      "start": 306.469,
+      "duration": 0.0,
+      "text": "simply sample from this model and now we"
+    },
+    {
+      "start": 306.479,
+      "duration": 0.0,
+      "text": "simply sample from this model and now we can<00:05:06.600><c> generate</c><00:05:07.000><c> data</c><00:05:07.880><c> uh</c><00:05:07.960><c> so</c><00:05:08.080><c> you</c><00:05:08.199><c> can</c><00:05:08.320><c> generate</c>"
+    },
+    {
+      "start": 308.83,
+      "duration": 0.0,
+      "text": "can generate data uh so you can generate"
+    },
+    {
+      "start": 308.84,
+      "duration": 0.0,
+      "text": "can generate data uh so you can generate sentences<00:05:09.840><c> uh</c><00:05:10.039><c> using</c><00:05:10.440><c> a</c><00:05:10.600><c> language</c>"
+    },
+    {
+      "start": 311.55,
+      "duration": 0.0,
+      "text": "sentences uh using a language"
+    },
+    {
+      "start": 311.56,
+      "duration": 0.0,
+      "text": "sentences uh using a language model<00:05:12.560><c> so</c><00:05:12.759><c> the</c><00:05:12.919><c> type</c><00:05:13.080><c> of</c><00:05:13.240><c> models</c><00:05:13.680><c> that</c><00:05:14.039><c> uh</c>"
+    },
+    {
+      "start": 314.15,
+      "duration": 0.0,
+      "text": "model so the type of models that uh"
+    },
+    {
+      "start": 314.16,
+      "duration": 0.0,
+      "text": "model so the type of models that uh people<00:05:14.440><c> are</c><00:05:14.639><c> all</c><00:05:14.880><c> currently</c><00:05:15.280><c> using</c><00:05:15.680><c> are</c><00:05:15.880><c> what</c>"
+    },
+    {
+      "start": 315.99,
+      "duration": 0.0,
+      "text": "people are all currently using are what"
+    },
+    {
+      "start": 316.0,
+      "duration": 0.0,
+      "text": "people are all currently using are what we<00:05:16.160><c> call</c><00:05:16.440><c> Auto</c><00:05:16.800><c> regressive</c><00:05:17.759><c> language</c><00:05:18.199><c> models</c>"
+    },
+    {
+      "start": 319.189,
+      "duration": 0.0,
+      "text": "we call Auto regressive language models"
+    },
+    {
+      "start": 319.199,
+      "duration": 0.0,
+      "text": "we call Auto regressive language models and<00:05:19.319><c> the</c><00:05:19.600><c> key</c><00:05:20.080><c> idea</c><00:05:20.440><c> of</c><00:05:20.600><c> autor</c><00:05:20.880><c> regressive</c>"
+    },
+    {
+      "start": 321.23,
+      "duration": 0.0,
+      "text": "and the key idea of autor regressive"
+    },
+    {
+      "start": 321.24,
+      "duration": 0.0,
+      "text": "and the key idea of autor regressive language<00:05:21.560><c> models</c><00:05:22.080><c> is</c><00:05:22.240><c> that</c><00:05:22.400><c> you</c><00:05:22.560><c> take</c><00:05:23.199><c> this</c>"
+    },
+    {
+      "start": 323.43,
+      "duration": 0.0,
+      "text": "language models is that you take this"
+    },
+    {
+      "start": 323.44,
+      "duration": 0.0,
+      "text": "language models is that you take this distribution<00:05:24.360><c> over</c><00:05:24.759><c> words</c><00:05:25.639><c> and</c><00:05:25.800><c> you</c>"
+    },
+    {
+      "start": 326.029,
+      "duration": 0.0,
+      "text": "distribution over words and you"
+    },
+    {
+      "start": 326.039,
+      "duration": 0.0,
+      "text": "distribution over words and you basically<00:05:26.520><c> decompose</c><00:05:27.080><c> it</c><00:05:27.400><c> into</c><00:05:27.639><c> the</c><00:05:28.160><c> into</c><00:05:28.720><c> the</c>"
+    },
+    {
+      "start": 328.87,
+      "duration": 0.0,
+      "text": "basically decompose it into the into the"
+    },
+    {
+      "start": 328.88,
+      "duration": 0.0,
+      "text": "basically decompose it into the into the distribution<00:05:29.400><c> of</c><00:05:29.759><c> the</c><00:05:29.919><c> first</c><00:05:30.240><c> word</c><00:05:30.840><c> multiply</c>"
+    },
+    {
+      "start": 331.35,
+      "duration": 0.0,
+      "text": "distribution of the first word multiply"
+    },
+    {
+      "start": 331.36,
+      "duration": 0.0,
+      "text": "distribution of the first word multiply the<00:05:31.720><c> by</c><00:05:31.800><c> the</c><00:05:31.919><c> distribution</c><00:05:32.720><c> of</c><00:05:32.919><c> or</c><00:05:33.039><c> the</c>"
+    },
+    {
+      "start": 333.15,
+      "duration": 0.0,
+      "text": "the by the distribution of or the"
+    },
+    {
+      "start": 333.16,
+      "duration": 0.0,
+      "text": "the by the distribution of or the likelihood<00:05:33.560><c> of</c><00:05:33.639><c> the</c><00:05:33.759><c> distribution</c><00:05:34.360><c> of</c><00:05:34.600><c> the</c>"
+    },
+    {
+      "start": 334.79,
+      "duration": 0.0,
+      "text": "likelihood of the distribution of the"
+    },
+    {
+      "start": 334.8,
+      "duration": 0.0,
+      "text": "likelihood of the distribution of the second<00:05:35.080><c> word</c><00:05:35.479><c> given</c><00:05:35.759><c> the</c><00:05:35.919><c> first</c><00:05:36.199><c> word</c><00:05:36.919><c> uh</c>"
+    },
+    {
+      "start": 337.029,
+      "duration": 0.0,
+      "text": "second word given the first word uh"
+    },
+    {
+      "start": 337.039,
+      "duration": 0.0,
+      "text": "second word given the first word uh multiply<00:05:37.560><c> by</c><00:05:37.759><c> P</c><00:05:38.000><c> of</c><00:05:38.160><c> the</c><00:05:38.280><c> third</c><00:05:38.600><c> word</c><00:05:39.039><c> given</c>"
+    },
+    {
+      "start": 339.27,
+      "duration": 0.0,
+      "text": "multiply by P of the third word given"
+    },
+    {
+      "start": 339.28,
+      "duration": 0.0,
+      "text": "multiply by P of the third word given the<00:05:39.400><c> first</c><00:05:39.639><c> two</c><00:05:39.960><c> words</c><00:05:40.960><c> um</c><00:05:41.319><c> so</c><00:05:41.680><c> there's</c><00:05:41.840><c> no</c>"
+    },
+    {
+      "start": 342.029,
+      "duration": 0.0,
+      "text": "the first two words um so there's no"
+    },
+    {
+      "start": 342.039,
+      "duration": 0.0,
+      "text": "the first two words um so there's no approximation<00:05:42.680><c> here</c><00:05:42.919><c> this</c><00:05:43.000><c> is</c><00:05:43.160><c> just</c><00:05:43.280><c> the</c>"
+    },
+    {
+      "start": 343.39,
+      "duration": 0.0,
+      "text": "approximation here this is just the"
+    },
+    {
+      "start": 343.4,
+      "duration": 0.0,
+      "text": "approximation here this is just the chain<00:05:43.600><c> rule</c><00:05:43.800><c> of</c><00:05:43.919><c> probability</c><00:05:44.440><c> which</c><00:05:44.560><c> you</c>"
+    },
+    {
+      "start": 344.71,
+      "duration": 0.0,
+      "text": "chain rule of probability which you"
+    },
+    {
+      "start": 344.72,
+      "duration": 0.0,
+      "text": "chain rule of probability which you hopefully<00:05:45.080><c> all</c><00:05:45.240><c> know</c><00:05:45.440><c> about</c><00:05:46.080><c> uh</c><00:05:46.240><c> really</c><00:05:46.400><c> no</c>"
+    },
+    {
+      "start": 346.67,
+      "duration": 0.0,
+      "text": "hopefully all know about uh really no"
+    },
+    {
+      "start": 346.68,
+      "duration": 0.0,
+      "text": "hopefully all know about uh really no approximation<00:05:47.240><c> this</c><00:05:47.360><c> is</c><00:05:47.520><c> just</c><00:05:47.720><c> one</c><00:05:48.000><c> way</c><00:05:48.199><c> of</c>"
+    },
+    {
+      "start": 348.35,
+      "duration": 0.0,
+      "text": "approximation this is just one way of"
+    },
+    {
+      "start": 348.36,
+      "duration": 0.0,
+      "text": "approximation this is just one way of modeling<00:05:48.960><c> a</c>"
+    },
+    {
+      "start": 349.71,
+      "duration": 0.0,
+      "text": "modeling a"
+    },
+    {
+      "start": 349.72,
+      "duration": 0.0,
+      "text": "modeling a distribution<00:05:50.720><c> uh</c><00:05:50.840><c> so</c><00:05:51.080><c> slightly</c><00:05:51.400><c> more</c>"
+    },
+    {
+      "start": 351.59,
+      "duration": 0.0,
+      "text": "distribution uh so slightly more"
+    },
+    {
+      "start": 351.6,
+      "duration": 0.0,
+      "text": "distribution uh so slightly more concisely<00:05:52.039><c> you</c><00:05:52.120><c> can</c><00:05:52.319><c> write</c><00:05:52.440><c> it</c><00:05:52.560><c> as</c><00:05:52.680><c> a</c><00:05:52.880><c> product</c>"
+    },
+    {
+      "start": 353.27,
+      "duration": 0.0,
+      "text": "concisely you can write it as a product"
+    },
+    {
+      "start": 353.28,
+      "duration": 0.0,
+      "text": "concisely you can write it as a product of<00:05:53.840><c> U</c><00:05:54.360><c> of</c><00:05:54.600><c> PS</c><00:05:55.440><c> of</c><00:05:55.600><c> the</c><00:05:55.800><c> next</c><00:05:56.080><c> word</c><00:05:56.400><c> given</c>"
+    },
+    {
+      "start": 356.79,
+      "duration": 0.0,
+      "text": "of U of PS of the next word given"
+    },
+    {
+      "start": 356.8,
+      "duration": 0.0,
+      "text": "of U of PS of the next word given everything<00:05:57.160><c> which</c><00:05:57.360><c> happened</c><00:05:57.639><c> in</c><00:05:57.759><c> the</c><00:05:57.919><c> past</c><00:05:58.160><c> so</c>"
+    },
+    {
+      "start": 358.309,
+      "duration": 0.0,
+      "text": "everything which happened in the past so"
+    },
+    {
+      "start": 358.319,
+      "duration": 0.0,
+      "text": "everything which happened in the past so of<00:05:58.440><c> the</c><00:05:58.600><c> context</c><00:05:59.520><c> and</c><00:05:59.720><c> uh</c><00:05:59.800><c> so</c><00:06:00.080><c> this</c><00:06:00.319><c> this</c><00:06:00.440><c> is</c>"
+    },
+    {
+      "start": 360.55,
+      "duration": 0.0,
+      "text": "of the context and uh so this this is"
+    },
+    {
+      "start": 360.56,
+      "duration": 0.0,
+      "text": "of the context and uh so this this is what<00:06:00.680><c> we</c><00:06:00.840><c> call</c><00:06:01.080><c> Auto</c><00:06:01.360><c> regressive</c><00:06:01.759><c> language</c>"
+    },
+    {
+      "start": 362.07,
+      "duration": 0.0,
+      "text": "what we call Auto regressive language"
+    },
+    {
+      "start": 362.08,
+      "duration": 0.0,
+      "text": "what we call Auto regressive language models<00:06:02.840><c> again</c><00:06:03.080><c> this</c><00:06:03.240><c> is</c><00:06:03.759><c> really</c><00:06:04.080><c> not</c><00:06:04.280><c> the</c><00:06:04.440><c> only</c>"
+    },
+    {
+      "start": 364.749,
+      "duration": 0.0,
+      "text": "models again this is really not the only"
+    },
+    {
+      "start": 364.759,
+      "duration": 0.0,
+      "text": "models again this is really not the only way<00:06:04.960><c> of</c><00:06:05.319><c> modeling</c><00:06:05.720><c> distribution</c><00:06:06.280><c> this</c><00:06:06.400><c> is</c>"
+    },
+    {
+      "start": 366.55,
+      "duration": 0.0,
+      "text": "way of modeling distribution this is"
+    },
+    {
+      "start": 366.56,
+      "duration": 0.0,
+      "text": "way of modeling distribution this is just<00:06:06.800><c> one</c><00:06:07.080><c> way</c><00:06:07.880><c> uh</c><00:06:08.080><c> it</c><00:06:08.240><c> has</c><00:06:08.440><c> some</c><00:06:08.639><c> benefits</c><00:06:09.039><c> and</c>"
+    },
+    {
+      "start": 369.43,
+      "duration": 0.0,
+      "text": "just one way uh it has some benefits and"
+    },
+    {
+      "start": 369.44,
+      "duration": 0.0,
+      "text": "just one way uh it has some benefits and some<00:06:09.639><c> downsides</c><00:06:10.479><c> one</c><00:06:10.759><c> downside</c><00:06:11.280><c> of</c>"
+    },
+    {
+      "start": 371.39,
+      "duration": 0.0,
+      "text": "some downsides one downside of"
+    },
+    {
+      "start": 371.4,
+      "duration": 0.0,
+      "text": "some downsides one downside of autoaggressive<00:06:11.880><c> language</c><00:06:12.319><c> models</c><00:06:12.919><c> is</c><00:06:13.039><c> that</c>"
+    },
+    {
+      "start": 373.189,
+      "duration": 0.0,
+      "text": "autoaggressive language models is that"
+    },
+    {
+      "start": 373.199,
+      "duration": 0.0,
+      "text": "autoaggressive language models is that when<00:06:13.319><c> you</c><00:06:13.520><c> actually</c><00:06:13.759><c> sample</c><00:06:14.199><c> from</c><00:06:14.479><c> this</c>"
+    },
+    {
+      "start": 374.87,
+      "duration": 0.0,
+      "text": "when you actually sample from this"
+    },
+    {
+      "start": 374.88,
+      "duration": 0.0,
+      "text": "when you actually sample from this autoaggressive<00:06:15.400><c> language</c><00:06:15.800><c> model</c><00:06:16.039><c> you</c>"
+    },
+    {
+      "start": 376.11,
+      "duration": 0.0,
+      "text": "autoaggressive language model you"
+    },
+    {
+      "start": 376.12,
+      "duration": 0.0,
+      "text": "autoaggressive language model you basically<00:06:16.400><c> have</c><00:06:16.520><c> a</c><00:06:16.599><c> for</c><00:06:16.919><c> Loop</c><00:06:17.479><c> which</c>"
+    },
+    {
+      "start": 377.83,
+      "duration": 0.0,
+      "text": "basically have a for Loop which"
+    },
+    {
+      "start": 377.84,
+      "duration": 0.0,
+      "text": "basically have a for Loop which generates<00:06:18.240><c> the</c><00:06:18.400><c> next</c><00:06:18.680><c> word</c><00:06:19.240><c> then</c><00:06:19.680><c> conditions</c>"
+    },
+    {
+      "start": 380.27,
+      "duration": 0.0,
+      "text": "generates the next word then conditions"
+    },
+    {
+      "start": 380.28,
+      "duration": 0.0,
+      "text": "generates the next word then conditions on<00:06:20.520><c> that</c><00:06:20.720><c> next</c><00:06:20.960><c> word</c><00:06:21.400><c> and</c><00:06:21.479><c> then</c><00:06:21.639><c> regenerate</c><00:06:22.240><c> an</c>"
+    },
+    {
+      "start": 382.43,
+      "duration": 0.0,
+      "text": "on that next word and then regenerate an"
+    },
+    {
+      "start": 382.44,
+      "duration": 0.0,
+      "text": "on that next word and then regenerate an other<00:06:22.680><c> word</c><00:06:22.919><c> so</c><00:06:23.160><c> basically</c><00:06:23.800><c> if</c><00:06:23.880><c> you</c><00:06:24.000><c> have</c><00:06:24.120><c> a</c>"
+    },
+    {
+      "start": 384.23,
+      "duration": 0.0,
+      "text": "other word so basically if you have a"
+    },
+    {
+      "start": 384.24,
+      "duration": 0.0,
+      "text": "other word so basically if you have a longer<00:06:24.560><c> sentence</c><00:06:24.919><c> that</c><00:06:25.039><c> you</c><00:06:25.120><c> want</c><00:06:25.240><c> to</c>"
+    },
+    {
+      "start": 385.35,
+      "duration": 0.0,
+      "text": "longer sentence that you want to"
+    },
+    {
+      "start": 385.36,
+      "duration": 0.0,
+      "text": "longer sentence that you want to generate<00:06:26.080><c> you</c><00:06:26.319><c> it</c><00:06:26.479><c> takes</c><00:06:26.720><c> more</c><00:06:26.960><c> time</c><00:06:27.120><c> to</c>"
+    },
+    {
+      "start": 387.27,
+      "duration": 0.0,
+      "text": "generate you it takes more time to"
+    },
+    {
+      "start": 387.28,
+      "duration": 0.0,
+      "text": "generate you it takes more time to generate<00:06:27.639><c> it</c><00:06:28.240><c> uh</c><00:06:28.319><c> so</c><00:06:28.479><c> there</c><00:06:28.560><c> are</c><00:06:28.720><c> some</c>"
+    },
+    {
+      "start": 388.909,
+      "duration": 0.0,
+      "text": "generate it uh so there are some"
+    },
+    {
+      "start": 388.919,
+      "duration": 0.0,
+      "text": "generate it uh so there are some downsides<00:06:29.759><c> of</c><00:06:29.960><c> this</c><00:06:30.120><c> current</c><00:06:30.440><c> Paradigm</c><00:06:30.880><c> but</c>"
+    },
+    {
+      "start": 391.029,
+      "duration": 0.0,
+      "text": "downsides of this current Paradigm but"
+    },
+    {
+      "start": 391.039,
+      "duration": 0.0,
+      "text": "downsides of this current Paradigm but that's<00:06:31.280><c> what</c><00:06:31.520><c> we</c><00:06:32.080><c> currently</c><00:06:32.599><c> have</c><00:06:32.800><c> so</c><00:06:32.919><c> I'm</c>"
+    },
+    {
+      "start": 393.029,
+      "duration": 0.0,
+      "text": "that's what we currently have so I'm"
+    },
+    {
+      "start": 393.039,
+      "duration": 0.0,
+      "text": "that's what we currently have so I'm going<00:06:33.120><c> to</c><00:06:33.280><c> talk</c><00:06:33.440><c> about</c><00:06:33.680><c> this</c>"
+    },
+    {
+      "start": 394.87,
+      "duration": 0.0,
+      "text": "going to talk about this"
+    },
+    {
+      "start": 394.88,
+      "duration": 0.0,
+      "text": "going to talk about this one<00:06:35.880><c> uh</c><00:06:36.080><c> great</c><00:06:36.680><c> so</c><00:06:36.880><c> Auto</c><00:06:37.160><c> regressive</c><00:06:37.560><c> language</c>"
+    },
+    {
+      "start": 397.83,
+      "duration": 0.0,
+      "text": "one uh great so Auto regressive language"
+    },
+    {
+      "start": 397.84,
+      "duration": 0.0,
+      "text": "one uh great so Auto regressive language models<00:06:38.280><c> at</c><00:06:38.360><c> a</c><00:06:38.520><c> high</c><00:06:38.720><c> level</c><00:06:39.479><c> um</c><00:06:39.800><c> what</c><00:06:39.919><c> the</c><00:06:40.160><c> task</c>"
+    },
+    {
+      "start": 400.469,
+      "duration": 0.0,
+      "text": "models at a high level um what the task"
+    },
+    {
+      "start": 400.479,
+      "duration": 0.0,
+      "text": "models at a high level um what the task of<00:06:40.720><c> autoregressive</c><00:06:41.280><c> language</c><00:06:41.560><c> model</c><00:06:41.880><c> is</c><00:06:42.120><c> is</c>"
+    },
+    {
+      "start": 402.23,
+      "duration": 0.0,
+      "text": "of autoregressive language model is is"
+    },
+    {
+      "start": 402.24,
+      "duration": 0.0,
+      "text": "of autoregressive language model is is simply<00:06:42.479><c> predicting</c><00:06:42.840><c> the</c><00:06:43.000><c> next</c><00:06:43.199><c> word</c><00:06:43.440><c> as</c><00:06:43.520><c> I</c>"
+    },
+    {
+      "start": 403.67,
+      "duration": 0.0,
+      "text": "simply predicting the next word as I"
+    },
+    {
+      "start": 403.68,
+      "duration": 0.0,
+      "text": "simply predicting the next word as I just<00:06:43.800><c> said</c><00:06:44.199><c> so</c><00:06:44.319><c> if</c><00:06:44.400><c> you</c><00:06:44.520><c> have</c><00:06:44.599><c> a</c><00:06:44.720><c> sentence</c><00:06:45.039><c> like</c>"
+    },
+    {
+      "start": 405.15,
+      "duration": 0.0,
+      "text": "just said so if you have a sentence like"
+    },
+    {
+      "start": 405.16,
+      "duration": 0.0,
+      "text": "just said so if you have a sentence like she<00:06:45.479><c> likely</c><00:06:45.919><c> prefers</c><00:06:46.919><c> uh</c><00:06:47.160><c> one</c><00:06:47.440><c> potential</c><00:06:48.039><c> next</c>"
+    },
+    {
+      "start": 408.27,
+      "duration": 0.0,
+      "text": "she likely prefers uh one potential next"
+    },
+    {
+      "start": 408.28,
+      "duration": 0.0,
+      "text": "she likely prefers uh one potential next word<00:06:48.560><c> might</c><00:06:48.759><c> be</c><00:06:49.520><c> dogs</c><00:06:50.520><c> and</c><00:06:50.680><c> the</c><00:06:51.120><c> the</c><00:06:51.240><c> way</c><00:06:51.440><c> we</c><00:06:51.560><c> do</c>"
+    },
+    {
+      "start": 411.71,
+      "duration": 0.0,
+      "text": "word might be dogs and the the way we do"
+    },
+    {
+      "start": 411.72,
+      "duration": 0.0,
+      "text": "word might be dogs and the the way we do it<00:06:52.120><c> is</c><00:06:52.319><c> that</c><00:06:52.520><c> we</c><00:06:52.960><c> first</c><00:06:53.440><c> tokenize</c><00:06:54.440><c> so</c><00:06:54.639><c> you</c><00:06:54.800><c> take</c>"
+    },
+    {
+      "start": 415.029,
+      "duration": 0.0,
+      "text": "it is that we first tokenize so you take"
+    },
+    {
+      "start": 415.039,
+      "duration": 0.0,
+      "text": "it is that we first tokenize so you take these<00:06:55.240><c> words</c><00:06:55.800><c> or</c><00:06:56.080><c> subwords</c><00:06:56.680><c> you</c><00:06:56.840><c> tokenize</c>"
+    },
+    {
+      "start": 417.43,
+      "duration": 0.0,
+      "text": "these words or subwords you tokenize"
+    },
+    {
+      "start": 417.44,
+      "duration": 0.0,
+      "text": "these words or subwords you tokenize them<00:06:58.160><c> um</c><00:06:58.479><c> and</c><00:06:58.639><c> then</c><00:06:58.759><c> you</c><00:06:58.919><c> give</c><00:06:59.080><c> an</c><00:06:59.280><c> IDE</c><00:06:59.960><c> for</c>"
+    },
+    {
+      "start": 420.15,
+      "duration": 0.0,
+      "text": "them um and then you give an IDE for"
+    },
+    {
+      "start": 420.16,
+      "duration": 0.0,
+      "text": "them um and then you give an IDE for each<00:07:00.360><c> token</c><00:07:00.680><c> so</c><00:07:00.840><c> here</c><00:07:00.919><c> you</c><00:07:01.039><c> have</c><00:07:01.199><c> 1</c><00:07:01.360><c> 2</c><00:07:02.000><c> three</c><00:07:03.000><c> uh</c>"
+    },
+    {
+      "start": 423.189,
+      "duration": 0.0,
+      "text": "each token so here you have 1 2 three uh"
+    },
+    {
+      "start": 423.199,
+      "duration": 0.0,
+      "text": "each token so here you have 1 2 three uh then<00:07:03.360><c> you</c><00:07:03.720><c> pass</c><00:07:03.879><c> it</c><00:07:04.039><c> through</c><00:07:04.240><c> this</c><00:07:04.440><c> black</c><00:07:04.720><c> box</c>"
+    },
+    {
+      "start": 425.029,
+      "duration": 0.0,
+      "text": "then you pass it through this black box"
+    },
+    {
+      "start": 425.039,
+      "duration": 0.0,
+      "text": "then you pass it through this black box as<00:07:05.160><c> I</c><00:07:05.319><c> already</c><00:07:05.560><c> said</c><00:07:05.759><c> we're</c><00:07:05.919><c> not</c><00:07:06.000><c> going</c><00:07:06.120><c> to</c>"
+    },
+    {
+      "start": 426.189,
+      "duration": 0.0,
+      "text": "as I already said we're not going to"
+    },
+    {
+      "start": 426.199,
+      "duration": 0.0,
+      "text": "as I already said we're not going to talk<00:07:06.360><c> about</c><00:07:06.520><c> the</c><00:07:06.639><c> architecture</c><00:07:07.280><c> you</c><00:07:07.440><c> just</c>"
+    },
+    {
+      "start": 427.67,
+      "duration": 0.0,
+      "text": "talk about the architecture you just"
+    },
+    {
+      "start": 427.68,
+      "duration": 0.0,
+      "text": "talk about the architecture you just pass<00:07:07.879><c> it</c><00:07:08.560><c> pass</c><00:07:08.800><c> it</c><00:07:08.960><c> through</c><00:07:09.120><c> a</c><00:07:09.240><c> model</c><00:07:10.120><c> and</c><00:07:10.240><c> you</c>"
+    },
+    {
+      "start": 430.43,
+      "duration": 0.0,
+      "text": "pass it pass it through a model and you"
+    },
+    {
+      "start": 430.44,
+      "duration": 0.0,
+      "text": "pass it pass it through a model and you then<00:07:10.680><c> get</c><00:07:11.080><c> a</c><00:07:11.280><c> distribution</c><00:07:12.160><c> a</c><00:07:12.360><c> probability</c>"
+    },
+    {
+      "start": 432.909,
+      "duration": 0.0,
+      "text": "then get a distribution a probability"
+    },
+    {
+      "start": 432.919,
+      "duration": 0.0,
+      "text": "then get a distribution a probability distribution<00:07:13.720><c> over</c><00:07:14.000><c> the</c><00:07:14.199><c> next</c><00:07:14.520><c> word</c><00:07:14.879><c> over</c><00:07:15.080><c> the</c>"
+    },
+    {
+      "start": 435.189,
+      "duration": 0.0,
+      "text": "distribution over the next word over the"
+    },
+    {
+      "start": 435.199,
+      "duration": 0.0,
+      "text": "distribution over the next word over the next<00:07:15.879><c> token</c><00:07:16.879><c> and</c><00:07:17.080><c> then</c><00:07:17.520><c> you</c><00:07:17.800><c> sample</c><00:07:18.720><c> uh</c><00:07:18.840><c> from</c>"
+    },
+    {
+      "start": 439.07,
+      "duration": 0.0,
+      "text": "next token and then you sample uh from"
+    },
+    {
+      "start": 439.08,
+      "duration": 0.0,
+      "text": "next token and then you sample uh from this<00:07:19.240><c> distribution</c><00:07:20.039><c> you</c><00:07:20.199><c> get</c><00:07:20.319><c> a</c><00:07:20.479><c> new</c><00:07:20.720><c> token</c>"
+    },
+    {
+      "start": 441.469,
+      "duration": 0.0,
+      "text": "this distribution you get a new token"
+    },
+    {
+      "start": 441.479,
+      "duration": 0.0,
+      "text": "this distribution you get a new token and<00:07:21.599><c> then</c><00:07:21.720><c> you</c><00:07:21.879><c> DET</c><00:07:22.160><c> tokenize</c><00:07:22.840><c> so</c><00:07:22.960><c> you</c><00:07:23.080><c> get</c><00:07:23.160><c> a</c>"
+    },
+    {
+      "start": 443.27,
+      "duration": 0.0,
+      "text": "and then you DET tokenize so you get a"
+    },
+    {
+      "start": 443.28,
+      "duration": 0.0,
+      "text": "and then you DET tokenize so you get a new<00:07:23.440><c> ID</c><00:07:23.800><c> you</c><00:07:23.960><c> then</c><00:07:24.080><c> DET</c><00:07:24.319><c> toonize</c><00:07:24.720><c> and</c><00:07:25.240><c> that's</c>"
+    },
+    {
+      "start": 445.39,
+      "duration": 0.0,
+      "text": "new ID you then DET toonize and that's"
+    },
+    {
+      "start": 445.4,
+      "duration": 0.0,
+      "text": "new ID you then DET toonize and that's how<00:07:25.560><c> you</c><00:07:25.720><c> basically</c><00:07:26.080><c> sample</c><00:07:26.479><c> from</c><00:07:26.879><c> a</c><00:07:27.039><c> language</c>"
+    },
+    {
+      "start": 447.43,
+      "duration": 0.0,
+      "text": "how you basically sample from a language"
+    },
+    {
+      "start": 447.44,
+      "duration": 0.0,
+      "text": "how you basically sample from a language model<00:07:28.319><c> uh</c><00:07:28.440><c> one</c><00:07:28.599><c> thing</c><00:07:28.759><c> which</c><00:07:28.840><c> is</c><00:07:28.960><c> important</c><00:07:29.240><c> to</c>"
+    },
+    {
+      "start": 449.309,
+      "duration": 0.0,
+      "text": "model uh one thing which is important to"
+    },
+    {
+      "start": 449.319,
+      "duration": 0.0,
+      "text": "model uh one thing which is important to not<00:07:29.720><c> is</c><00:07:29.800><c> that</c><00:07:29.960><c> the</c><00:07:30.120><c> last</c><00:07:30.319><c> two</c><00:07:30.560><c> TS</c><00:07:31.000><c> uh</c><00:07:31.120><c> two</c><00:07:31.319><c> steps</c>"
+    },
+    {
+      "start": 451.589,
+      "duration": 0.0,
+      "text": "not is that the last two TS uh two steps"
+    },
+    {
+      "start": 451.599,
+      "duration": 0.0,
+      "text": "not is that the last two TS uh two steps are<00:07:31.759><c> actually</c><00:07:32.039><c> only</c><00:07:32.360><c> need</c><00:07:32.599><c> needed</c><00:07:32.960><c> during</c>"
+    },
+    {
+      "start": 453.27,
+      "duration": 0.0,
+      "text": "are actually only need needed during"
+    },
+    {
+      "start": 453.28,
+      "duration": 0.0,
+      "text": "are actually only need needed during inference<00:07:34.240><c> uh</c><00:07:34.360><c> when</c><00:07:34.479><c> you</c><00:07:34.560><c> do</c><00:07:34.759><c> training</c><00:07:35.400><c> you</c>"
+    },
+    {
+      "start": 455.55,
+      "duration": 0.0,
+      "text": "inference uh when you do training you"
+    },
+    {
+      "start": 455.56,
+      "duration": 0.0,
+      "text": "inference uh when you do training you just<00:07:35.720><c> need</c><00:07:35.879><c> to</c><00:07:36.080><c> predict</c><00:07:36.840><c> uh</c><00:07:36.919><c> the</c><00:07:37.039><c> most</c><00:07:37.240><c> likely</c>"
+    },
+    {
+      "start": 457.589,
+      "duration": 0.0,
+      "text": "just need to predict uh the most likely"
+    },
+    {
+      "start": 457.599,
+      "duration": 0.0,
+      "text": "just need to predict uh the most likely token<00:07:38.039><c> and</c><00:07:38.120><c> you</c><00:07:38.199><c> can</c><00:07:38.400><c> just</c><00:07:38.599><c> compare</c><00:07:39.080><c> to</c><00:07:39.240><c> the</c>"
+    },
+    {
+      "start": 459.39,
+      "duration": 0.0,
+      "text": "token and you can just compare to the"
+    },
+    {
+      "start": 459.4,
+      "duration": 0.0,
+      "text": "token and you can just compare to the real<00:07:39.680><c> token</c><00:07:40.199><c> which</c><00:07:40.400><c> happen</c><00:07:40.680><c> in</c><00:07:40.879><c> practice</c><00:07:41.479><c> and</c>"
+    },
+    {
+      "start": 461.589,
+      "duration": 0.0,
+      "text": "real token which happen in practice and"
+    },
+    {
+      "start": 461.599,
+      "duration": 0.0,
+      "text": "real token which happen in practice and then<00:07:41.759><c> you</c><00:07:42.400><c> basically</c><00:07:42.840><c> change</c><00:07:43.240><c> the</c><00:07:43.400><c> weights</c><00:07:43.800><c> of</c>"
+    },
+    {
+      "start": 463.909,
+      "duration": 0.0,
+      "text": "then you basically change the weights of"
+    },
+    {
+      "start": 463.919,
+      "duration": 0.0,
+      "text": "then you basically change the weights of your<00:07:44.080><c> model</c><00:07:44.639><c> to</c><00:07:44.840><c> increase</c><00:07:45.199><c> the</c><00:07:45.319><c> probability</c>"
+    },
+    {
+      "start": 465.749,
+      "duration": 0.0,
+      "text": "your model to increase the probability"
+    },
+    {
+      "start": 465.759,
+      "duration": 0.0,
+      "text": "your model to increase the probability of<00:07:45.840><c> generating</c><00:07:46.319><c> that</c>"
+    },
+    {
+      "start": 467.99,
+      "duration": 0.0,
+      "text": "of generating that"
+    },
+    {
+      "start": 468.0,
+      "duration": 0.0,
+      "text": "of generating that token<00:07:49.000><c> um</c><00:07:49.560><c> great</c><00:07:50.120><c> so</c><00:07:50.440><c> autoaggressive</c><00:07:50.960><c> neural</c>"
+    },
+    {
+      "start": 471.39,
+      "duration": 0.0,
+      "text": "token um great so autoaggressive neural"
+    },
+    {
+      "start": 471.4,
+      "duration": 0.0,
+      "text": "token um great so autoaggressive neural language<00:07:51.759><c> models</c><00:07:52.400><c> so</c><00:07:52.560><c> to</c><00:07:52.680><c> be</c><00:07:52.800><c> slightly</c><00:07:53.120><c> more</c>"
+    },
+    {
+      "start": 473.23,
+      "duration": 0.0,
+      "text": "language models so to be slightly more"
+    },
+    {
+      "start": 473.24,
+      "duration": 0.0,
+      "text": "language models so to be slightly more specific<00:07:53.720><c> still</c><00:07:53.960><c> without</c><00:07:54.280><c> talking</c><00:07:54.479><c> about</c><00:07:54.639><c> the</c>"
+    },
+    {
+      "start": 474.909,
+      "duration": 0.0,
+      "text": "specific still without talking about the"
+    },
+    {
+      "start": 474.919,
+      "duration": 0.0,
+      "text": "specific still without talking about the architecture<00:07:55.919><c> uh</c><00:07:56.039><c> the</c><00:07:56.159><c> first</c><00:07:56.360><c> thing</c><00:07:56.520><c> we</c><00:07:56.639><c> do</c><00:07:57.120><c> is</c>"
+    },
+    {
+      "start": 477.23,
+      "duration": 0.0,
+      "text": "architecture uh the first thing we do is"
+    },
+    {
+      "start": 477.24,
+      "duration": 0.0,
+      "text": "architecture uh the first thing we do is that<00:07:57.440><c> we</c><00:07:57.639><c> have</c><00:07:57.879><c> all</c><00:07:58.039><c> of</c><00:07:58.240><c> these</c><00:07:58.680><c> oh</c><00:07:58.840><c> sorry</c><00:07:59.159><c> yes</c>"
+    },
+    {
+      "start": 479.55,
+      "duration": 0.0,
+      "text": "that we have all of these oh sorry yes"
+    },
+    {
+      "start": 479.56,
+      "duration": 0.0,
+      "text": "that we have all of these oh sorry yes on<00:07:59.759><c> the</c><00:08:00.000><c> previous</c><00:08:00.400><c> slide</c><00:08:00.960><c> when</c><00:08:01.159><c> you're</c>"
+    },
+    {
+      "start": 481.51,
+      "duration": 0.0,
+      "text": "on the previous slide when you're"
+    },
+    {
+      "start": 481.52,
+      "duration": 0.0,
+      "text": "on the previous slide when you're predicting<00:08:01.960><c> the</c><00:08:02.120><c> probability</c><00:08:02.520><c> of</c><00:08:02.560><c> the</c><00:08:02.720><c> next</c>"
+    },
+    {
+      "start": 482.909,
+      "duration": 0.0,
+      "text": "predicting the probability of the next"
+    },
+    {
+      "start": 482.919,
+      "duration": 0.0,
+      "text": "predicting the probability of the next tokens<00:08:03.240><c> does</c><00:08:03.360><c> this</c><00:08:03.520><c> mean</c><00:08:03.680><c> that</c><00:08:03.800><c> your</c><00:08:04.000><c> final</c>"
+    },
+    {
+      "start": 484.55,
+      "duration": 0.0,
+      "text": "tokens does this mean that your final"
+    },
+    {
+      "start": 484.56,
+      "duration": 0.0,
+      "text": "tokens does this mean that your final like<00:08:04.879><c> output</c><00:08:05.319><c> VOR</c><00:08:05.759><c> has</c><00:08:05.879><c> to</c><00:08:06.000><c> be</c><00:08:06.280><c> the</c><00:08:06.360><c> same</c>"
+    },
+    {
+      "start": 486.67,
+      "duration": 0.0,
+      "text": "like output VOR has to be the same"
+    },
+    {
+      "start": 486.68,
+      "duration": 0.0,
+      "text": "like output VOR has to be the same dimensionality<00:08:07.680><c> as</c><00:08:07.840><c> the</c><00:08:08.000><c> number</c><00:08:08.240><c> of</c><00:08:08.400><c> tokens</c>"
+    },
+    {
+      "start": 488.749,
+      "duration": 0.0,
+      "text": "dimensionality as the number of tokens"
+    },
+    {
+      "start": 488.759,
+      "duration": 0.0,
+      "text": "dimensionality as the number of tokens that<00:08:08.840><c> you</c><00:08:09.039><c> have</c><00:08:09.440><c> yes</c><00:08:10.440><c> how</c><00:08:10.520><c> do</c><00:08:10.599><c> you</c><00:08:10.759><c> deal</c><00:08:11.000><c> with</c>"
+    },
+    {
+      "start": 491.189,
+      "duration": 0.0,
+      "text": "that you have yes how do you deal with"
+    },
+    {
+      "start": 491.199,
+      "duration": 0.0,
+      "text": "that you have yes how do you deal with like<00:08:11.400><c> if</c><00:08:11.520><c> you</c><00:08:11.960><c> have</c><00:08:12.280><c> more</c><00:08:12.560><c> to</c><00:08:12.879><c> like</c><00:08:13.000><c> if</c><00:08:13.080><c> you're</c>"
+    },
+    {
+      "start": 493.469,
+      "duration": 0.0,
+      "text": "like if you have more to like if you're"
+    },
+    {
+      "start": 493.479,
+      "duration": 0.0,
+      "text": "like if you have more to like if you're adding<00:08:13.800><c> more</c><00:08:14.000><c> tokens</c><00:08:14.280><c> to</c><00:08:14.400><c> your</c><00:08:14.520><c> cor</c><00:08:15.479><c> something</c>"
+    },
+    {
+      "start": 496.309,
+      "duration": 0.0,
+      "text": "adding more tokens to your cor something"
+    },
+    {
+      "start": 496.319,
+      "duration": 0.0,
+      "text": "adding more tokens to your cor something yeah<00:08:16.759><c> so</c><00:08:16.879><c> we're</c><00:08:17.000><c> going</c><00:08:17.080><c> to</c><00:08:17.199><c> talk</c><00:08:17.360><c> about</c>"
+    },
+    {
+      "start": 497.67,
+      "duration": 0.0,
+      "text": "yeah so we're going to talk about"
+    },
+    {
+      "start": 497.68,
+      "duration": 0.0,
+      "text": "yeah so we're going to talk about tokenization<00:08:18.599><c> actually</c><00:08:18.960><c> later</c><00:08:19.599><c> uh</c><00:08:19.720><c> so</c><00:08:19.840><c> you</c>"
+    },
+    {
+      "start": 499.909,
+      "duration": 0.0,
+      "text": "tokenization actually later uh so you"
+    },
+    {
+      "start": 499.919,
+      "duration": 0.0,
+      "text": "tokenization actually later uh so you will<00:08:20.120><c> get</c><00:08:20.360><c> some</c><00:08:20.599><c> sense</c><00:08:20.840><c> of</c><00:08:21.080><c> this</c><00:08:21.720><c> you</c>"
+    },
+    {
+      "start": 501.99,
+      "duration": 0.0,
+      "text": "will get some sense of this you"
+    },
+    {
+      "start": 502.0,
+      "duration": 0.0,
+      "text": "will get some sense of this you basically<00:08:22.520><c> can</c><00:08:22.840><c> deal</c><00:08:23.479><c> with</c><00:08:23.680><c> adding</c><00:08:24.039><c> new</c>"
+    },
+    {
+      "start": 504.27,
+      "duration": 0.0,
+      "text": "basically can deal with adding new"
+    },
+    {
+      "start": 504.28,
+      "duration": 0.0,
+      "text": "basically can deal with adding new tokens<00:08:25.000><c> I</c><00:08:25.159><c> am</c><00:08:25.360><c> I'm</c><00:08:25.560><c> kind</c><00:08:25.680><c> of</c><00:08:25.840><c> exaggerating</c>"
+    },
+    {
+      "start": 506.39,
+      "duration": 0.0,
+      "text": "tokens I am I'm kind of exaggerating"
+    },
+    {
+      "start": 506.4,
+      "duration": 0.0,
+      "text": "tokens I am I'm kind of exaggerating there<00:08:26.520><c> are</c><00:08:26.720><c> methods</c><00:08:27.000><c> for</c><00:08:27.159><c> doing</c><00:08:27.360><c> it</c><00:08:27.520><c> but</c>"
+    },
+    {
+      "start": 507.629,
+      "duration": 0.0,
+      "text": "there are methods for doing it but"
+    },
+    {
+      "start": 507.639,
+      "duration": 0.0,
+      "text": "there are methods for doing it but essentially<00:08:28.039><c> people</c><00:08:28.280><c> don't</c><00:08:28.479><c> do</c><00:08:28.639><c> it</c><00:08:29.319><c> um</c><00:08:29.879><c> so</c>"
+    },
+    {
+      "start": 510.589,
+      "duration": 0.0,
+      "text": "essentially people don't do it um so"
+    },
+    {
+      "start": 510.599,
+      "duration": 0.0,
+      "text": "essentially people don't do it um so it's<00:08:30.919><c> really</c><00:08:31.199><c> important</c><00:08:31.560><c> to</c><00:08:31.759><c> think</c><00:08:32.200><c> about</c><00:08:32.399><c> how</c>"
+    },
+    {
+      "start": 512.509,
+      "duration": 0.0,
+      "text": "it's really important to think about how"
+    },
+    {
+      "start": 512.519,
+      "duration": 0.0,
+      "text": "it's really important to think about how you<00:08:32.640><c> tokenize</c><00:08:33.120><c> your</c><00:08:33.240><c> text</c><00:08:33.479><c> and</c><00:08:33.560><c> that's</c><00:08:33.680><c> why</c>"
+    },
+    {
+      "start": 513.79,
+      "duration": 0.0,
+      "text": "you tokenize your text and that's why"
+    },
+    {
+      "start": 513.8,
+      "duration": 0.0,
+      "text": "you tokenize your text and that's why we'll<00:08:34.000><c> talk</c><00:08:34.200><c> about</c><00:08:34.399><c> that</c><00:08:34.599><c> later</c><00:08:35.560><c> but</c><00:08:35.680><c> it's</c><00:08:35.800><c> a</c>"
+    },
+    {
+      "start": 515.99,
+      "duration": 0.0,
+      "text": "we'll talk about that later but it's a"
+    },
+    {
+      "start": 516.0,
+      "duration": 0.0,
+      "text": "we'll talk about that later but it's a very<00:08:36.159><c> good</c><00:08:36.320><c> point</c><00:08:36.479><c> to</c><00:08:36.599><c> notice</c><00:08:37.000><c> that</c><00:08:37.120><c> you</c>"
+    },
+    {
+      "start": 517.23,
+      "duration": 0.0,
+      "text": "very good point to notice that you"
+    },
+    {
+      "start": 517.24,
+      "duration": 0.0,
+      "text": "very good point to notice that you basically<00:08:37.640><c> the</c><00:08:37.800><c> vocabulary</c><00:08:38.320><c> size</c><00:08:38.519><c> so</c><00:08:38.680><c> the</c>"
+    },
+    {
+      "start": 518.75,
+      "duration": 0.0,
+      "text": "basically the vocabulary size so the"
+    },
+    {
+      "start": 518.76,
+      "duration": 0.0,
+      "text": "basically the vocabulary size so the number<00:08:38.959><c> of</c><00:08:39.080><c> tokens</c><00:08:39.399><c> that</c><00:08:39.519><c> you</c><00:08:39.680><c> have</c><00:08:40.039><c> is</c>"
+    },
+    {
+      "start": 520.149,
+      "duration": 0.0,
+      "text": "number of tokens that you have is"
+    },
+    {
+      "start": 520.159,
+      "duration": 0.0,
+      "text": "number of tokens that you have is essentially<00:08:40.599><c> the</c><00:08:40.719><c> output</c><00:08:41.560><c> of</c><00:08:41.760><c> your</c><00:08:42.200><c> uh</c>"
+    },
+    {
+      "start": 522.31,
+      "duration": 0.0,
+      "text": "essentially the output of your uh"
+    },
+    {
+      "start": 522.32,
+      "duration": 0.0,
+      "text": "essentially the output of your uh language<00:08:42.719><c> model</c><00:08:43.200><c> so</c><00:08:43.360><c> it's</c><00:08:43.560><c> actually</c><00:08:43.800><c> pretty</c>"
+    },
+    {
+      "start": 524.23,
+      "duration": 0.0,
+      "text": "language model so it's actually pretty"
+    },
+    {
+      "start": 524.24,
+      "duration": 0.0,
+      "text": "language model so it's actually pretty pretty"
+    },
+    {
+      "start": 525.19,
+      "duration": 0.0,
+      "text": "pretty"
+    },
+    {
+      "start": 525.2,
+      "duration": 0.0,
+      "text": "pretty large<00:08:46.200><c> okay</c><00:08:46.320><c> so</c><00:08:46.480><c> autoaggressive</c><00:08:47.000><c> new</c>"
+    },
+    {
+      "start": 527.43,
+      "duration": 0.0,
+      "text": "large okay so autoaggressive new"
+    },
+    {
+      "start": 527.44,
+      "duration": 0.0,
+      "text": "large okay so autoaggressive new language<00:08:47.800><c> models</c><00:08:48.800><c> first</c><00:08:49.040><c> thing</c><00:08:49.160><c> you</c><00:08:49.320><c> do</c><00:08:49.600><c> is</c>"
+    },
+    {
+      "start": 529.71,
+      "duration": 0.0,
+      "text": "language models first thing you do is"
+    },
+    {
+      "start": 529.72,
+      "duration": 0.0,
+      "text": "language models first thing you do is that<00:08:49.880><c> you</c><00:08:50.040><c> take</c><00:08:50.360><c> every</c><00:08:50.600><c> word</c><00:08:50.800><c> or</c><00:08:50.959><c> every</c><00:08:51.200><c> token</c>"
+    },
+    {
+      "start": 532.03,
+      "duration": 0.0,
+      "text": "that you take every word or every token"
+    },
+    {
+      "start": 532.04,
+      "duration": 0.0,
+      "text": "that you take every word or every token you<00:08:52.360><c> embed</c><00:08:52.800><c> them</c><00:08:53.000><c> so</c><00:08:53.160><c> you</c><00:08:53.279><c> get</c><00:08:53.560><c> a</c><00:08:54.000><c> um</c><00:08:54.800><c> some</c>"
+    },
+    {
+      "start": 535.19,
+      "duration": 0.0,
+      "text": "you embed them so you get a um some"
+    },
+    {
+      "start": 535.2,
+      "duration": 0.0,
+      "text": "you embed them so you get a um some Vector<00:08:55.519><c> representation</c><00:08:56.120><c> for</c><00:08:56.320><c> each</c><00:08:56.480><c> of</c><00:08:56.640><c> these</c>"
+    },
+    {
+      "start": 537.03,
+      "duration": 0.0,
+      "text": "Vector representation for each of these"
+    },
+    {
+      "start": 537.04,
+      "duration": 0.0,
+      "text": "Vector representation for each of these tokens<00:08:58.040><c> um</c><00:08:58.360><c> you</c><00:08:58.560><c> pass</c><00:08:58.800><c> them</c><00:08:58.959><c> through</c><00:08:59.160><c> some</c><00:08:59.440><c> ual</c>"
+    },
+    {
+      "start": 539.63,
+      "duration": 0.0,
+      "text": "tokens um you pass them through some ual"
+    },
+    {
+      "start": 539.64,
+      "duration": 0.0,
+      "text": "tokens um you pass them through some ual Network<00:08:59.959><c> as</c><00:09:00.040><c> we</c><00:09:00.160><c> said</c><00:09:00.320><c> it's</c><00:09:00.440><c> a</c><00:09:00.560><c> Transformer</c>"
+    },
+    {
+      "start": 541.47,
+      "duration": 0.0,
+      "text": "Network as we said it's a Transformer"
+    },
+    {
+      "start": 541.48,
+      "duration": 0.0,
+      "text": "Network as we said it's a Transformer then<00:09:01.640><c> you</c><00:09:01.760><c> get</c><00:09:01.920><c> a</c><00:09:02.240><c> representation</c><00:09:03.240><c> for</c><00:09:03.720><c> all</c>"
+    },
+    {
+      "start": 543.949,
+      "duration": 0.0,
+      "text": "then you get a representation for all"
+    },
+    {
+      "start": 543.959,
+      "duration": 0.0,
+      "text": "then you get a representation for all the<00:09:04.160><c> word</c><00:09:04.800><c> in</c><00:09:05.079><c> all</c><00:09:05.240><c> the</c><00:09:05.399><c> words</c><00:09:05.760><c> in</c><00:09:05.880><c> the</c><00:09:06.079><c> context</c>"
+    },
+    {
+      "start": 546.63,
+      "duration": 0.0,
+      "text": "the word in all the words in the context"
+    },
+    {
+      "start": 546.64,
+      "duration": 0.0,
+      "text": "the word in all the words in the context so<00:09:06.800><c> it's</c><00:09:06.959><c> basically</c><00:09:07.279><c> representation</c><00:09:08.000><c> of</c><00:09:08.120><c> the</c>"
+    },
+    {
+      "start": 548.269,
+      "duration": 0.0,
+      "text": "so it's basically representation of the"
+    },
+    {
+      "start": 548.279,
+      "duration": 0.0,
+      "text": "so it's basically representation of the entire<00:09:08.839><c> sentence</c><00:09:09.839><c> uh</c><00:09:10.000><c> you</c><00:09:10.200><c> pass</c><00:09:10.360><c> it</c><00:09:10.519><c> through</c><00:09:10.680><c> a</c>"
+    },
+    {
+      "start": 550.79,
+      "duration": 0.0,
+      "text": "entire sentence uh you pass it through a"
+    },
+    {
+      "start": 550.8,
+      "duration": 0.0,
+      "text": "entire sentence uh you pass it through a linear<00:09:11.120><c> layer</c><00:09:11.800><c> as</c><00:09:11.920><c> you</c><00:09:12.120><c> just</c><00:09:12.279><c> said</c><00:09:12.720><c> to</c>"
+    },
+    {
+      "start": 552.99,
+      "duration": 0.0,
+      "text": "linear layer as you just said to"
+    },
+    {
+      "start": 553.0,
+      "duration": 0.0,
+      "text": "linear layer as you just said to basically<00:09:13.839><c> map</c><00:09:14.120><c> it</c><00:09:14.279><c> to</c><00:09:14.519><c> the</c><00:09:15.000><c> number</c><00:09:15.680><c> so</c><00:09:15.880><c> that</c>"
+    },
+    {
+      "start": 556.03,
+      "duration": 0.0,
+      "text": "basically map it to the number so that"
+    },
+    {
+      "start": 556.04,
+      "duration": 0.0,
+      "text": "basically map it to the number so that the<00:09:16.200><c> output</c><00:09:16.560><c> the</c><00:09:16.640><c> number</c><00:09:17.000><c> of</c><00:09:17.240><c> outputs</c><00:09:17.640><c> is</c><00:09:17.720><c> the</c>"
+    },
+    {
+      "start": 557.87,
+      "duration": 0.0,
+      "text": "the output the number of outputs is the"
+    },
+    {
+      "start": 557.88,
+      "duration": 0.0,
+      "text": "the output the number of outputs is the number<00:09:18.079><c> of</c><00:09:18.360><c> tokens</c><00:09:19.360><c> uh</c><00:09:19.640><c> you</c><00:09:19.839><c> then</c><00:09:20.040><c> pass</c><00:09:20.200><c> it</c>"
+    },
+    {
+      "start": 560.35,
+      "duration": 0.0,
+      "text": "number of tokens uh you then pass it"
+    },
+    {
+      "start": 560.36,
+      "duration": 0.0,
+      "text": "number of tokens uh you then pass it through<00:09:20.519><c> some</c><00:09:20.720><c> soft</c><00:09:21.040><c> Max</c><00:09:21.640><c> and</c><00:09:21.760><c> you</c><00:09:21.920><c> basically</c>"
+    },
+    {
+      "start": 562.35,
+      "duration": 0.0,
+      "text": "through some soft Max and you basically"
+    },
+    {
+      "start": 562.36,
+      "duration": 0.0,
+      "text": "through some soft Max and you basically get<00:09:23.000><c> uh</c><00:09:23.200><c> probity</c><00:09:23.800><c> distribution</c><00:09:24.800><c> over</c><00:09:25.440><c> the</c>"
+    },
+    {
+      "start": 565.63,
+      "duration": 0.0,
+      "text": "get uh probity distribution over the"
+    },
+    {
+      "start": 565.64,
+      "duration": 0.0,
+      "text": "get uh probity distribution over the next<00:09:25.959><c> words</c><00:09:26.519><c> given</c><00:09:27.120><c> every</c><00:09:27.440><c> word</c><00:09:27.640><c> in</c><00:09:27.720><c> the</c>"
+    },
+    {
+      "start": 567.91,
+      "duration": 0.0,
+      "text": "next words given every word in the"
+    },
+    {
+      "start": 567.92,
+      "duration": 0.0,
+      "text": "next words given every word in the context"
+    },
+    {
+      "start": 570.55,
+      "duration": 0.0,
+      "text": "context"
+    },
+    {
+      "start": 570.56,
+      "duration": 0.0,
+      "text": "context and<00:09:30.680><c> the</c><00:09:30.760><c> law</c><00:09:31.079><c> that</c><00:09:31.200><c> you</c><00:09:31.360><c> use</c><00:09:31.880><c> is</c><00:09:32.079><c> basically</c>"
+    },
+    {
+      "start": 572.829,
+      "duration": 0.0,
+      "text": "and the law that you use is basically"
+    },
+    {
+      "start": 572.839,
+      "duration": 0.0,
+      "text": "and the law that you use is basically it's<00:09:33.040><c> essentially</c><00:09:33.399><c> a</c><00:09:33.560><c> task</c><00:09:33.800><c> of</c><00:09:34.000><c> classifying</c>"
+    },
+    {
+      "start": 574.55,
+      "duration": 0.0,
+      "text": "it's essentially a task of classifying"
+    },
+    {
+      "start": 574.56,
+      "duration": 0.0,
+      "text": "it's essentially a task of classifying the<00:09:34.720><c> next</c><00:09:35.000><c> token</c><00:09:35.279><c> so</c><00:09:35.440><c> it's</c><00:09:35.519><c> a</c><00:09:35.680><c> very</c><00:09:35.880><c> simple</c>"
+    },
+    {
+      "start": 576.269,
+      "duration": 0.0,
+      "text": "the next token so it's a very simple"
+    },
+    {
+      "start": 576.279,
+      "duration": 0.0,
+      "text": "the next token so it's a very simple kind<00:09:36.399><c> of</c><00:09:36.519><c> machine</c><00:09:36.800><c> learning</c><00:09:37.120><c> task</c><00:09:37.560><c> so</c><00:09:37.680><c> you</c><00:09:37.800><c> use</c>"
+    },
+    {
+      "start": 577.91,
+      "duration": 0.0,
+      "text": "kind of machine learning task so you use"
+    },
+    {
+      "start": 577.92,
+      "duration": 0.0,
+      "text": "kind of machine learning task so you use the<00:09:38.079><c> cross</c><00:09:38.279><c> entry</c><00:09:38.600><c> P</c><00:09:38.800><c> loss</c><00:09:39.399><c> where</c><00:09:39.560><c> you</c>"
+    },
+    {
+      "start": 579.829,
+      "duration": 0.0,
+      "text": "the cross entry P loss where you"
+    },
+    {
+      "start": 579.839,
+      "duration": 0.0,
+      "text": "the cross entry P loss where you basically<00:09:40.360><c> you</c><00:09:40.720><c> look</c><00:09:41.120><c> at</c><00:09:41.640><c> the</c><00:09:42.120><c> actual</c><00:09:43.040><c> Target</c>"
+    },
+    {
+      "start": 583.949,
+      "duration": 0.0,
+      "text": "basically you look at the actual Target"
+    },
+    {
+      "start": 583.959,
+      "duration": 0.0,
+      "text": "basically you look at the actual Target that<00:09:44.160><c> happened</c><00:09:44.480><c> which</c><00:09:44.600><c> is</c><00:09:44.680><c> a</c><00:09:44.800><c> target</c>"
+    },
+    {
+      "start": 585.03,
+      "duration": 0.0,
+      "text": "that happened which is a target"
+    },
+    {
+      "start": 585.04,
+      "duration": 0.0,
+      "text": "that happened which is a target distribution<00:09:45.640><c> which</c><00:09:45.760><c> is</c><00:09:45.839><c> a</c><00:09:46.000><c> one</c><00:09:46.240><c> hot</c><00:09:46.440><c> encoding</c>"
+    },
+    {
+      "start": 586.91,
+      "duration": 0.0,
+      "text": "distribution which is a one hot encoding"
+    },
+    {
+      "start": 586.92,
+      "duration": 0.0,
+      "text": "distribution which is a one hot encoding which<00:09:47.120><c> here</c><00:09:47.279><c> in</c><00:09:47.440><c> this</c><00:09:47.760><c> in</c><00:09:47.920><c> this</c><00:09:48.200><c> case</c><00:09:48.480><c> says</c><00:09:49.040><c> I</c>"
+    },
+    {
+      "start": 589.19,
+      "duration": 0.0,
+      "text": "which here in this in this case says I"
+    },
+    {
+      "start": 589.2,
+      "duration": 0.0,
+      "text": "which here in this in this case says I saw<00:09:49.560><c> uh</c><00:09:50.040><c> the</c><00:09:50.200><c> real</c><00:09:50.440><c> word</c><00:09:50.680><c> that</c><00:09:50.839><c> happened</c><00:09:51.240><c> is</c>"
+    },
+    {
+      "start": 591.47,
+      "duration": 0.0,
+      "text": "saw uh the real word that happened is"
+    },
+    {
+      "start": 591.48,
+      "duration": 0.0,
+      "text": "saw uh the real word that happened is cat<00:09:51.880><c> so</c><00:09:52.000><c> that's</c><00:09:52.120><c> a</c><00:09:52.240><c> one</c><00:09:52.480><c> hot</c><00:09:53.200><c> um</c><00:09:53.920><c> distribution</c>"
+    },
+    {
+      "start": 594.91,
+      "duration": 0.0,
+      "text": "cat so that's a one hot um distribution"
+    },
+    {
+      "start": 594.92,
+      "duration": 0.0,
+      "text": "cat so that's a one hot um distribution over<00:09:55.279><c> cat</c><00:09:55.680><c> and</c><00:09:55.880><c> here</c><00:09:56.079><c> this</c><00:09:56.200><c> is</c><00:09:56.360><c> the</c><00:09:56.600><c> actual</c><00:09:57.560><c> uh</c>"
+    },
+    {
+      "start": 597.71,
+      "duration": 0.0,
+      "text": "over cat and here this is the actual uh"
+    },
+    {
+      "start": 597.72,
+      "duration": 0.0,
+      "text": "over cat and here this is the actual uh do<00:09:57.800><c> you</c><00:09:57.920><c> see</c><00:09:58.079><c> my</c><00:09:58.200><c> mouse</c><00:09:58.480><c> oh</c><00:09:58.640><c> yeah</c><00:09:58.839><c> this</c><00:09:58.959><c> is</c><00:09:59.040><c> the</c>"
+    },
+    {
+      "start": 599.35,
+      "duration": 0.0,
+      "text": "do you see my mouse oh yeah this is the"
+    },
+    {
+      "start": 599.36,
+      "duration": 0.0,
+      "text": "do you see my mouse oh yeah this is the distribtion<00:09:59.680><c> that</c><00:09:59.760><c> you</c><00:09:59.880><c> generated</c><00:10:00.600><c> and</c>"
+    },
+    {
+      "start": 600.71,
+      "duration": 0.0,
+      "text": "distribtion that you generated and"
+    },
+    {
+      "start": 600.72,
+      "duration": 0.0,
+      "text": "distribtion that you generated and basically<00:10:01.000><c> you</c><00:10:01.120><c> do</c><00:10:01.279><c> cross</c><00:10:01.519><c> entropy</c><00:10:01.959><c> which</c>"
+    },
+    {
+      "start": 602.19,
+      "duration": 0.0,
+      "text": "basically you do cross entropy which"
+    },
+    {
+      "start": 602.2,
+      "duration": 0.0,
+      "text": "basically you do cross entropy which really<00:10:02.480><c> just</c><00:10:02.760><c> increases</c><00:10:03.240><c> the</c><00:10:03.360><c> probability</c><00:10:03.720><c> of</c>"
+    },
+    {
+      "start": 603.79,
+      "duration": 0.0,
+      "text": "really just increases the probability of"
+    },
+    {
+      "start": 603.8,
+      "duration": 0.0,
+      "text": "really just increases the probability of generating<00:10:04.240><c> cat</c><00:10:04.440><c> and</c><00:10:04.600><c> decreases</c><00:10:05.360><c> all</c><00:10:05.560><c> the</c><00:10:05.880><c> the</c>"
+    },
+    {
+      "start": 605.949,
+      "duration": 0.0,
+      "text": "generating cat and decreases all the the"
+    },
+    {
+      "start": 605.959,
+      "duration": 0.0,
+      "text": "generating cat and decreases all the the probility<00:10:06.279><c> of</c><00:10:06.360><c> generating</c><00:10:06.800><c> all</c><00:10:06.959><c> the</c><00:10:07.040><c> other</c>"
+    },
+    {
+      "start": 607.35,
+      "duration": 0.0,
+      "text": "probility of generating all the other"
+    },
+    {
+      "start": 607.36,
+      "duration": 0.0,
+      "text": "probility of generating all the other tokens<00:10:08.200><c> one</c><00:10:08.399><c> thing</c><00:10:08.560><c> to</c><00:10:08.720><c> notice</c><00:10:09.519><c> is</c><00:10:09.720><c> that</c><00:10:09.959><c> as</c>"
+    },
+    {
+      "start": 610.069,
+      "duration": 0.0,
+      "text": "tokens one thing to notice is that as"
+    },
+    {
+      "start": 610.079,
+      "duration": 0.0,
+      "text": "tokens one thing to notice is that as you<00:10:10.200><c> all</c><00:10:10.399><c> know</c><00:10:10.839><c> again</c><00:10:11.560><c> uh</c><00:10:11.760><c> this</c><00:10:12.079><c> is</c><00:10:12.519><c> just</c>"
+    },
+    {
+      "start": 612.79,
+      "duration": 0.0,
+      "text": "you all know again uh this is just"
+    },
+    {
+      "start": 612.8,
+      "duration": 0.0,
+      "text": "you all know again uh this is just equivalent<00:10:13.360><c> to</c><00:10:13.600><c> maximizing</c><00:10:14.240><c> the</c><00:10:14.399><c> text</c><00:10:14.600><c> log</c>"
+    },
+    {
+      "start": 614.91,
+      "duration": 0.0,
+      "text": "equivalent to maximizing the text log"
+    },
+    {
+      "start": 614.92,
+      "duration": 0.0,
+      "text": "equivalent to maximizing the text log like<00:10:15.279><c> the</c><00:10:15.440><c> text</c><00:10:15.680><c> log</c><00:10:15.920><c> likelihood</c><00:10:16.640><c> because</c><00:10:16.760><c> you</c>"
+    },
+    {
+      "start": 616.87,
+      "duration": 0.0,
+      "text": "like the text log likelihood because you"
+    },
+    {
+      "start": 616.88,
+      "duration": 0.0,
+      "text": "like the text log likelihood because you can<00:10:17.040><c> just</c><00:10:17.480><c> rewrite</c><00:10:18.160><c> the</c><00:10:18.720><c> the</c><00:10:19.320><c> max</c><00:10:19.800><c> over</c><00:10:20.240><c> the</c>"
+    },
+    {
+      "start": 620.47,
+      "duration": 0.0,
+      "text": "can just rewrite the the max over the"
+    },
+    {
+      "start": 620.48,
+      "duration": 0.0,
+      "text": "can just rewrite the the max over the probability<00:10:21.480><c> of</c><00:10:21.920><c> um</c><00:10:22.120><c> this</c><00:10:22.279><c> autoregressive</c>"
+    },
+    {
+      "start": 622.91,
+      "duration": 0.0,
+      "text": "probability of um this autoregressive"
+    },
+    {
+      "start": 622.92,
+      "duration": 0.0,
+      "text": "probability of um this autoregressive language<00:10:23.160><c> moding</c><00:10:23.600><c> task</c><00:10:24.360><c> as</c><00:10:24.560><c> just</c><00:10:24.720><c> being</c><00:10:25.000><c> this</c>"
+    },
+    {
+      "start": 625.23,
+      "duration": 0.0,
+      "text": "language moding task as just being this"
+    },
+    {
+      "start": 625.24,
+      "duration": 0.0,
+      "text": "language moding task as just being this minimum<00:10:26.079><c> over</c><00:10:26.600><c> I</c><00:10:26.720><c> just</c><00:10:26.880><c> added</c><00:10:27.160><c> the</c><00:10:27.320><c> log</c><00:10:27.680><c> here</c>"
+    },
+    {
+      "start": 627.87,
+      "duration": 0.0,
+      "text": "minimum over I just added the log here"
+    },
+    {
+      "start": 627.88,
+      "duration": 0.0,
+      "text": "minimum over I just added the log here and<00:10:28.120><c> minus</c><00:10:28.880><c> which</c><00:10:29.000><c> is</c><00:10:29.399><c> just</c><00:10:29.519><c> the</c><00:10:29.600><c> minimum</c><00:10:29.959><c> of</c>"
+    },
+    {
+      "start": 630.069,
+      "duration": 0.0,
+      "text": "and minus which is just the minimum of"
+    },
+    {
+      "start": 630.079,
+      "duration": 0.0,
+      "text": "and minus which is just the minimum of the<00:10:30.200><c> loss</c><00:10:30.480><c> which</c><00:10:30.560><c> is</c><00:10:30.640><c> the</c><00:10:30.760><c> cross</c><00:10:30.959><c> enty</c><00:10:31.399><c> loss</c><00:10:31.600><c> so</c>"
+    },
+    {
+      "start": 631.75,
+      "duration": 0.0,
+      "text": "the loss which is the cross enty loss so"
+    },
+    {
+      "start": 631.76,
+      "duration": 0.0,
+      "text": "the loss which is the cross enty loss so basically<00:10:32.079><c> minimizing</c><00:10:32.560><c> the</c><00:10:32.680><c> loss</c><00:10:33.120><c> is</c><00:10:33.240><c> the</c>"
+    },
+    {
+      "start": 633.35,
+      "duration": 0.0,
+      "text": "basically minimizing the loss is the"
+    },
+    {
+      "start": 633.36,
+      "duration": 0.0,
+      "text": "basically minimizing the loss is the same<00:10:33.560><c> thing</c><00:10:33.760><c> as</c><00:10:33.959><c> maximizing</c><00:10:34.519><c> the</c><00:10:34.640><c> likelihood</c>"
+    },
+    {
+      "start": 635.47,
+      "duration": 0.0,
+      "text": "same thing as maximizing the likelihood"
+    },
+    {
+      "start": 635.48,
+      "duration": 0.0,
+      "text": "same thing as maximizing the likelihood of<00:10:35.639><c> your</c><00:10:35.920><c> text</c><00:10:36.920><c> any</c><00:10:37.120><c> question</c>"
+    },
+    {
+      "start": 642.32,
+      "duration": 0.0,
+      "text": "questions"
+    },
+    {
+      "start": 643.91,
+      "duration": 0.0,
+      "text": "questions"
+    },
+    {
+      "start": 643.92,
+      "duration": 0.0,
+      "text": "questions okay"
+    },
+    {
+      "start": 645.829,
+      "duration": 0.0,
+      "text": "okay"
+    },
+    {
+      "start": 645.839,
+      "duration": 0.0,
+      "text": "okay tokenizer<00:10:46.839><c> um</c><00:10:47.200><c> so</c><00:10:47.959><c> this</c><00:10:48.079><c> is</c><00:10:48.279><c> one</c><00:10:48.519><c> thing</c><00:10:48.720><c> that</c>"
+    },
+    {
+      "start": 648.87,
+      "duration": 0.0,
+      "text": "tokenizer um so this is one thing that"
+    },
+    {
+      "start": 648.88,
+      "duration": 0.0,
+      "text": "tokenizer um so this is one thing that people<00:10:49.120><c> usually</c><00:10:49.399><c> don't</c><00:10:49.680><c> talk</c><00:10:50.079><c> that</c><00:10:50.240><c> much</c>"
+    },
+    {
+      "start": 650.47,
+      "duration": 0.0,
+      "text": "people usually don't talk that much"
+    },
+    {
+      "start": 650.48,
+      "duration": 0.0,
+      "text": "people usually don't talk that much about<00:10:50.880><c> tokenizers</c><00:10:51.680><c> are</c><00:10:52.040><c> extremely</c><00:10:52.680><c> important</c>"
+    },
+    {
+      "start": 653.389,
+      "duration": 0.0,
+      "text": "about tokenizers are extremely important"
+    },
+    {
+      "start": 653.399,
+      "duration": 0.0,
+      "text": "about tokenizers are extremely important uh<00:10:53.519><c> so</c><00:10:53.639><c> it's</c><00:10:53.760><c> really</c><00:10:53.959><c> important</c><00:10:54.279><c> that</c><00:10:54.399><c> you</c>"
+    },
+    {
+      "start": 654.55,
+      "duration": 0.0,
+      "text": "uh so it's really important that you"
+    },
+    {
+      "start": 654.56,
+      "duration": 0.0,
+      "text": "uh so it's really important that you kind<00:10:54.680><c> of</c><00:10:55.079><c> understand</c><00:10:55.240><c> at</c><00:10:55.360><c> least</c><00:10:56.200><c> uh</c><00:10:56.399><c> what</c><00:10:56.519><c> they</c>"
+    },
+    {
+      "start": 656.629,
+      "duration": 0.0,
+      "text": "kind of understand at least uh what they"
+    },
+    {
+      "start": 656.639,
+      "duration": 0.0,
+      "text": "kind of understand at least uh what they do<00:10:56.800><c> at</c><00:10:56.920><c> a</c><00:10:57.040><c> high</c><00:10:57.240><c> level</c><00:10:58.040><c> so</c><00:10:58.279><c> why</c><00:10:58.440><c> do</c><00:10:58.519><c> we</c><00:10:58.639><c> need</c>"
+    },
+    {
+      "start": 658.829,
+      "duration": 0.0,
+      "text": "do at a high level so why do we need"
+    },
+    {
+      "start": 658.839,
+      "duration": 0.0,
+      "text": "do at a high level so why do we need token<00:10:59.560><c> in</c><00:10:59.639><c> the</c><00:10:59.760><c> first</c><00:11:00.040><c> place</c><00:11:01.040><c> uh</c><00:11:01.279><c> first</c><00:11:01.680><c> it's</c>"
+    },
+    {
+      "start": 661.87,
+      "duration": 0.0,
+      "text": "token in the first place uh first it's"
+    },
+    {
+      "start": 661.88,
+      "duration": 0.0,
+      "text": "token in the first place uh first it's more<00:11:02.079><c> General</c><00:11:02.440><c> than</c><00:11:02.639><c> words</c><00:11:03.079><c> so</c><00:11:03.360><c> one</c><00:11:03.639><c> simple</c>"
+    },
+    {
+      "start": 664.03,
+      "duration": 0.0,
+      "text": "more General than words so one simple"
+    },
+    {
+      "start": 664.04,
+      "duration": 0.0,
+      "text": "more General than words so one simple thing<00:11:04.240><c> that</c><00:11:04.320><c> you</c><00:11:04.480><c> might</c><00:11:04.639><c> think</c><00:11:04.880><c> is</c><00:11:05.120><c> oh</c><00:11:05.279><c> we're</c>"
+    },
+    {
+      "start": 665.47,
+      "duration": 0.0,
+      "text": "thing that you might think is oh we're"
+    },
+    {
+      "start": 665.48,
+      "duration": 0.0,
+      "text": "thing that you might think is oh we're just<00:11:05.600><c> going</c><00:11:05.720><c> to</c><00:11:05.839><c> take</c><00:11:06.079><c> every</c><00:11:06.279><c> word</c><00:11:06.519><c> that</c><00:11:06.639><c> we</c>"
+    },
+    {
+      "start": 666.71,
+      "duration": 0.0,
+      "text": "just going to take every word that we"
+    },
+    {
+      "start": 666.72,
+      "duration": 0.0,
+      "text": "just going to take every word that we will<00:11:07.000><c> have</c><00:11:07.440><c> you</c><00:11:07.680><c> just</c><00:11:07.839><c> say</c><00:11:08.279><c> every</c><00:11:08.560><c> word</c><00:11:08.880><c> is</c><00:11:09.000><c> a</c>"
+    },
+    {
+      "start": 669.15,
+      "duration": 0.0,
+      "text": "will have you just say every word is a"
+    },
+    {
+      "start": 669.16,
+      "duration": 0.0,
+      "text": "will have you just say every word is a new<00:11:09.440><c> is</c><00:11:09.519><c> a</c><00:11:09.639><c> token</c><00:11:09.920><c> in</c><00:11:10.040><c> its</c><00:11:10.160><c> own</c><00:11:11.040><c> um</c><00:11:11.399><c> but</c><00:11:11.600><c> then</c>"
+    },
+    {
+      "start": 671.949,
+      "duration": 0.0,
+      "text": "new is a token in its own um but then"
+    },
+    {
+      "start": 671.959,
+      "duration": 0.0,
+      "text": "new is a token in its own um but then what<00:11:12.120><c> happens</c><00:11:12.440><c> is</c><00:11:12.600><c> if</c><00:11:12.720><c> there's</c><00:11:12.880><c> a</c><00:11:13.040><c> typo</c><00:11:13.560><c> in</c>"
+    },
+    {
+      "start": 673.67,
+      "duration": 0.0,
+      "text": "what happens is if there's a typo in"
+    },
+    {
+      "start": 673.68,
+      "duration": 0.0,
+      "text": "what happens is if there's a typo in your<00:11:13.920><c> word</c><00:11:14.920><c> then</c><00:11:15.040><c> you</c><00:11:15.279><c> might</c><00:11:15.519><c> not</c><00:11:15.760><c> have</c><00:11:16.000><c> any</c>"
+    },
+    {
+      "start": 676.19,
+      "duration": 0.0,
+      "text": "your word then you might not have any"
+    },
+    {
+      "start": 676.2,
+      "duration": 0.0,
+      "text": "your word then you might not have any token<00:11:16.760><c> associated</c><00:11:17.760><c> with</c><00:11:18.160><c> this</c><00:11:18.600><c> this</c><00:11:18.839><c> word</c>"
+    },
+    {
+      "start": 679.11,
+      "duration": 0.0,
+      "text": "token associated with this this word"
+    },
+    {
+      "start": 679.12,
+      "duration": 0.0,
+      "text": "token associated with this this word with<00:11:19.320><c> a</c><00:11:19.440><c> typo</c><00:11:20.040><c> and</c><00:11:20.160><c> then</c><00:11:20.279><c> you</c><00:11:20.399><c> don't</c><00:11:20.639><c> know</c><00:11:20.839><c> how</c>"
+    },
+    {
+      "start": 680.949,
+      "duration": 0.0,
+      "text": "with a typo and then you don't know how"
+    },
+    {
+      "start": 680.959,
+      "duration": 0.0,
+      "text": "with a typo and then you don't know how to<00:11:21.200><c> actually</c><00:11:21.560><c> pass</c><00:11:21.959><c> this</c><00:11:22.079><c> word</c><00:11:22.279><c> with</c><00:11:22.480><c> a</c><00:11:22.600><c> typo</c>"
+    },
+    {
+      "start": 682.949,
+      "duration": 0.0,
+      "text": "to actually pass this word with a typo"
+    },
+    {
+      "start": 682.959,
+      "duration": 0.0,
+      "text": "to actually pass this word with a typo into<00:11:23.160><c> a</c><00:11:23.279><c> large</c><00:11:23.560><c> language</c><00:11:23.880><c> model</c><00:11:24.600><c> so</c><00:11:24.760><c> what</c><00:11:24.880><c> do</c>"
+    },
+    {
+      "start": 684.949,
+      "duration": 0.0,
+      "text": "into a large language model so what do"
+    },
+    {
+      "start": 684.959,
+      "duration": 0.0,
+      "text": "into a large language model so what do you<00:11:25.079><c> do</c><00:11:25.320><c> next</c><00:11:25.800><c> and</c><00:11:26.000><c> also</c><00:11:26.560><c> even</c><00:11:26.760><c> if</c><00:11:26.880><c> you</c><00:11:27.000><c> think</c>"
+    },
+    {
+      "start": 687.15,
+      "duration": 0.0,
+      "text": "you do next and also even if you think"
+    },
+    {
+      "start": 687.16,
+      "duration": 0.0,
+      "text": "you do next and also even if you think about<00:11:27.399><c> words</c><00:11:27.920><c> words</c><00:11:28.160><c> is</c><00:11:28.240><c> a</c><00:11:28.480><c> very</c><00:11:28.800><c> like</c><00:11:29.440><c> words</c>"
+    },
+    {
+      "start": 689.829,
+      "duration": 0.0,
+      "text": "about words words is a very like words"
+    },
+    {
+      "start": 689.839,
+      "duration": 0.0,
+      "text": "about words words is a very like words are<00:11:30.040><c> fine</c><00:11:30.279><c> with</c><00:11:30.480><c> like</c><00:11:30.600><c> Latin</c><00:11:30.959><c> based</c><00:11:31.360><c> languages</c>"
+    },
+    {
+      "start": 692.35,
+      "duration": 0.0,
+      "text": "are fine with like Latin based languages"
+    },
+    {
+      "start": 692.36,
+      "duration": 0.0,
+      "text": "are fine with like Latin based languages uh<00:11:32.519><c> but</c><00:11:32.680><c> if</c><00:11:32.800><c> you</c><00:11:32.959><c> think</c><00:11:33.200><c> about</c><00:11:33.519><c> a</c><00:11:33.680><c> language</c>"
+    },
+    {
+      "start": 693.99,
+      "duration": 0.0,
+      "text": "uh but if you think about a language"
+    },
+    {
+      "start": 694.0,
+      "duration": 0.0,
+      "text": "uh but if you think about a language like<00:11:34.200><c> taii</c><00:11:34.839><c> you</c><00:11:35.000><c> won't</c><00:11:35.320><c> have</c><00:11:35.480><c> a</c><00:11:35.600><c> simple</c><00:11:35.920><c> way</c><00:11:36.079><c> of</c>"
+    },
+    {
+      "start": 696.23,
+      "duration": 0.0,
+      "text": "like taii you won't have a simple way of"
+    },
+    {
+      "start": 696.24,
+      "duration": 0.0,
+      "text": "like taii you won't have a simple way of tokenizing<00:11:36.800><c> by</c><00:11:37.000><c> spaces</c><00:11:37.440><c> because</c><00:11:37.519><c> there</c><00:11:37.600><c> are</c>"
+    },
+    {
+      "start": 697.71,
+      "duration": 0.0,
+      "text": "tokenizing by spaces because there are"
+    },
+    {
+      "start": 697.72,
+      "duration": 0.0,
+      "text": "tokenizing by spaces because there are no<00:11:37.880><c> spaces</c><00:11:38.279><c> between</c><00:11:38.600><c> words</c><00:11:39.480><c> um</c><00:11:39.760><c> so</c><00:11:40.040><c> really</c><00:11:40.880><c> uh</c>"
+    },
+    {
+      "start": 701.03,
+      "duration": 0.0,
+      "text": "no spaces between words um so really uh"
+    },
+    {
+      "start": 701.04,
+      "duration": 0.0,
+      "text": "no spaces between words um so really uh tokens<00:11:41.360><c> are</c><00:11:41.519><c> much</c><00:11:41.680><c> more</c><00:11:41.839><c> General</c><00:11:42.399><c> Than</c><00:11:42.800><c> Words</c>"
+    },
+    {
+      "start": 703.79,
+      "duration": 0.0,
+      "text": "tokens are much more General Than Words"
+    },
+    {
+      "start": 703.8,
+      "duration": 0.0,
+      "text": "tokens are much more General Than Words first<00:11:44.000><c> thing</c><00:11:44.240><c> second</c><00:11:44.480><c> thing</c><00:11:44.600><c> that</c><00:11:44.720><c> you</c><00:11:44.839><c> might</c>"
+    },
+    {
+      "start": 705.03,
+      "duration": 0.0,
+      "text": "first thing second thing that you might"
+    },
+    {
+      "start": 705.04,
+      "duration": 0.0,
+      "text": "first thing second thing that you might think<00:11:45.480><c> is</c><00:11:45.639><c> that</c><00:11:45.800><c> you</c><00:11:46.040><c> might</c><00:11:46.279><c> tokenize</c><00:11:47.279><c> every</c>"
+    },
+    {
+      "start": 707.59,
+      "duration": 0.0,
+      "text": "think is that you might tokenize every"
+    },
+    {
+      "start": 707.6,
+      "duration": 0.0,
+      "text": "think is that you might tokenize every sentence<00:11:48.240><c> character</c><00:11:48.639><c> by</c><00:11:48.880><c> character</c><00:11:49.399><c> you</c>"
+    },
+    {
+      "start": 709.55,
+      "duration": 0.0,
+      "text": "sentence character by character you"
+    },
+    {
+      "start": 709.56,
+      "duration": 0.0,
+      "text": "sentence character by character you might<00:11:49.720><c> say</c><00:11:49.959><c> a</c><00:11:50.240><c> is</c><00:11:50.399><c> one</c><00:11:50.600><c> token</c><00:11:50.959><c> b</c><00:11:51.200><c> is</c><00:11:51.360><c> another</c>"
+    },
+    {
+      "start": 711.67,
+      "duration": 0.0,
+      "text": "might say a is one token b is another"
+    },
+    {
+      "start": 711.68,
+      "duration": 0.0,
+      "text": "might say a is one token b is another token<00:11:52.639><c> uh</c><00:11:52.880><c> that</c><00:11:53.040><c> would</c><00:11:53.320><c> actually</c><00:11:53.600><c> work</c><00:11:54.040><c> and</c>"
+    },
+    {
+      "start": 714.19,
+      "duration": 0.0,
+      "text": "token uh that would actually work and"
+    },
+    {
+      "start": 714.2,
+      "duration": 0.0,
+      "text": "token uh that would actually work and probably<00:11:54.560><c> very</c><00:11:54.800><c> well</c><00:11:55.440><c> the</c><00:11:55.639><c> issue</c><00:11:55.920><c> is</c><00:11:56.040><c> that</c>"
+    },
+    {
+      "start": 716.19,
+      "duration": 0.0,
+      "text": "probably very well the issue is that"
+    },
+    {
+      "start": 716.2,
+      "duration": 0.0,
+      "text": "probably very well the issue is that then<00:11:56.360><c> your</c><00:11:56.519><c> sequence</c><00:11:56.959><c> becomes</c><00:11:57.440><c> super</c><00:11:57.760><c> long</c>"
+    },
+    {
+      "start": 718.43,
+      "duration": 0.0,
+      "text": "then your sequence becomes super long"
+    },
+    {
+      "start": 718.44,
+      "duration": 0.0,
+      "text": "then your sequence becomes super long and<00:11:58.600><c> as</c><00:11:58.720><c> you</c><00:11:59.240><c> probably</c><00:11:59.519><c> remember</c><00:11:59.920><c> from</c><00:12:00.079><c> the</c>"
+    },
+    {
+      "start": 720.23,
+      "duration": 0.0,
+      "text": "and as you probably remember from the"
+    },
+    {
+      "start": 720.24,
+      "duration": 0.0,
+      "text": "and as you probably remember from the lecture<00:12:00.600><c> on</c><00:12:00.800><c> on</c><00:12:01.200><c> Transformers</c><00:12:02.200><c> uh</c><00:12:02.360><c> the</c>"
+    },
+    {
+      "start": 722.87,
+      "duration": 0.0,
+      "text": "lecture on on Transformers uh the"
+    },
+    {
+      "start": 722.88,
+      "duration": 0.0,
+      "text": "lecture on on Transformers uh the complexity<00:12:03.880><c> uh</c><00:12:04.079><c> grows</c><00:12:04.519><c> quadratically</c><00:12:05.440><c> with</c>"
+    },
+    {
+      "start": 725.59,
+      "duration": 0.0,
+      "text": "complexity uh grows quadratically with"
+    },
+    {
+      "start": 725.6,
+      "duration": 0.0,
+      "text": "complexity uh grows quadratically with the<00:12:05.720><c> length</c><00:12:06.000><c> of</c><00:12:06.120><c> sequences</c><00:12:06.839><c> so</c><00:12:07.000><c> you</c><00:12:07.320><c> really</c>"
+    },
+    {
+      "start": 727.47,
+      "duration": 0.0,
+      "text": "the length of sequences so you really"
+    },
+    {
+      "start": 727.48,
+      "duration": 0.0,
+      "text": "the length of sequences so you really don't<00:12:07.680><c> want</c><00:12:07.760><c> to</c><00:12:07.959><c> have</c><00:12:08.160><c> a</c><00:12:08.279><c> super</c><00:12:08.600><c> long</c><00:12:08.959><c> sequence</c>"
+    },
+    {
+      "start": 729.949,
+      "duration": 0.0,
+      "text": "don't want to have a super long sequence"
+    },
+    {
+      "start": 729.959,
+      "duration": 0.0,
+      "text": "don't want to have a super long sequence um<00:12:10.120><c> so</c><00:12:10.440><c> tokenizers</c><00:12:11.440><c> basically</c><00:12:11.959><c> try</c><00:12:12.639><c> to</c><00:12:13.040><c> deal</c>"
+    },
+    {
+      "start": 733.31,
+      "duration": 0.0,
+      "text": "um so tokenizers basically try to deal"
+    },
+    {
+      "start": 733.32,
+      "duration": 0.0,
+      "text": "um so tokenizers basically try to deal with<00:12:13.519><c> those</c><00:12:13.720><c> two</c><00:12:14.000><c> problems</c><00:12:14.839><c> and</c><00:12:15.399><c> give</c><00:12:15.800><c> common</c>"
+    },
+    {
+      "start": 737.15,
+      "duration": 0.0,
+      "text": "with those two problems and give common"
+    },
+    {
+      "start": 737.16,
+      "duration": 0.0,
+      "text": "with those two problems and give common subsequences<00:12:18.160><c> a</c><00:12:18.360><c> certain</c><00:12:18.720><c> token</c><00:12:19.480><c> and</c><00:12:19.760><c> usually</c>"
+    },
+    {
+      "start": 740.15,
+      "duration": 0.0,
+      "text": "subsequences a certain token and usually"
+    },
+    {
+      "start": 740.16,
+      "duration": 0.0,
+      "text": "subsequences a certain token and usually how<00:12:20.279><c> you</c><00:12:20.360><c> should</c><00:12:20.600><c> be</c><00:12:20.760><c> think</c><00:12:20.959><c> about</c><00:12:21.199><c> is</c><00:12:21.600><c> around</c>"
+    },
+    {
+      "start": 742.47,
+      "duration": 0.0,
+      "text": "how you should be think about is around"
+    },
+    {
+      "start": 742.48,
+      "duration": 0.0,
+      "text": "how you should be think about is around uh<00:12:22.600><c> an</c><00:12:22.800><c> average</c><00:12:23.639><c> every</c><00:12:23.880><c> token</c><00:12:24.120><c> is</c><00:12:24.240><c> around</c>"
+    },
+    {
+      "start": 744.47,
+      "duration": 0.0,
+      "text": "uh an average every token is around"
+    },
+    {
+      "start": 744.48,
+      "duration": 0.0,
+      "text": "uh an average every token is around three<00:12:24.720><c> four</c><00:12:25.000><c> letters</c>"
+    },
+    {
+      "start": 746.87,
+      "duration": 0.0,
+      "text": "three four letters"
+    },
+    {
+      "start": 746.88,
+      "duration": 0.0,
+      "text": "three four letters um<00:12:27.880><c> and</c><00:12:28.399><c> there</c><00:12:28.519><c> are</c><00:12:28.639><c> many</c><00:12:28.800><c> algorithm</c><00:12:29.399><c> for</c>"
+    },
+    {
+      "start": 749.509,
+      "duration": 0.0,
+      "text": "um and there are many algorithm for"
+    },
+    {
+      "start": 749.519,
+      "duration": 0.0,
+      "text": "um and there are many algorithm for tokenization<00:12:30.160><c> I'll</c><00:12:30.320><c> just</c><00:12:30.440><c> talk</c><00:12:30.600><c> about</c><00:12:30.800><c> one</c><00:12:30.920><c> of</c>"
+    },
+    {
+      "start": 751.03,
+      "duration": 0.0,
+      "text": "tokenization I'll just talk about one of"
+    },
+    {
+      "start": 751.04,
+      "duration": 0.0,
+      "text": "tokenization I'll just talk about one of them<00:12:31.199><c> to</c><00:12:31.320><c> give</c><00:12:31.440><c> you</c><00:12:31.519><c> a</c><00:12:31.639><c> high</c><00:12:31.839><c> level</c><00:12:32.639><c> which</c><00:12:32.760><c> is</c>"
+    },
+    {
+      "start": 752.87,
+      "duration": 0.0,
+      "text": "them to give you a high level which is"
+    },
+    {
+      "start": 752.88,
+      "duration": 0.0,
+      "text": "them to give you a high level which is what<00:12:33.000><c> we</c><00:12:33.120><c> call</c><00:12:33.320><c> bite</c><00:12:33.600><c> P</c><00:12:33.760><c> en</c><00:12:33.880><c> coding</c><00:12:34.160><c> which</c><00:12:34.240><c> is</c>"
+    },
+    {
+      "start": 754.389,
+      "duration": 0.0,
+      "text": "what we call bite P en coding which is"
+    },
+    {
+      "start": 754.399,
+      "duration": 0.0,
+      "text": "what we call bite P en coding which is actually<00:12:34.680><c> pretty</c><00:12:34.959><c> common</c><00:12:35.360><c> one</c><00:12:35.480><c> of</c><00:12:35.600><c> the</c><00:12:35.720><c> two</c>"
+    },
+    {
+      "start": 755.91,
+      "duration": 0.0,
+      "text": "actually pretty common one of the two"
+    },
+    {
+      "start": 755.92,
+      "duration": 0.0,
+      "text": "actually pretty common one of the two most<00:12:36.120><c> common</c><00:12:36.880><c> tokenizers</c><00:12:37.880><c> and</c><00:12:37.959><c> the</c><00:12:38.120><c> way</c><00:12:38.279><c> that</c>"
+    },
+    {
+      "start": 758.35,
+      "duration": 0.0,
+      "text": "most common tokenizers and the way that"
+    },
+    {
+      "start": 758.36,
+      "duration": 0.0,
+      "text": "most common tokenizers and the way that you<00:12:38.480><c> train</c><00:12:38.680><c> a</c><00:12:38.959><c> tokenizer</c><00:12:39.959><c> is</c><00:12:40.160><c> that</c><00:12:40.399><c> first</c><00:12:40.600><c> you</c>"
+    },
+    {
+      "start": 760.71,
+      "duration": 0.0,
+      "text": "you train a tokenizer is that first you"
+    },
+    {
+      "start": 760.72,
+      "duration": 0.0,
+      "text": "you train a tokenizer is that first you start<00:12:41.000><c> with</c><00:12:41.120><c> a</c><00:12:41.279><c> very</c><00:12:41.560><c> large</c><00:12:42.000><c> Corpus</c><00:12:42.360><c> of</c><00:12:42.560><c> text</c>"
+    },
+    {
+      "start": 762.87,
+      "duration": 0.0,
+      "text": "start with a very large Corpus of text"
+    },
+    {
+      "start": 762.88,
+      "duration": 0.0,
+      "text": "start with a very large Corpus of text and<00:12:43.120><c> here</c><00:12:43.279><c> I'm</c><00:12:43.440><c> really</c><00:12:43.639><c> not</c><00:12:43.800><c> talking</c><00:12:44.079><c> about</c>"
+    },
+    {
+      "start": 764.23,
+      "duration": 0.0,
+      "text": "and here I'm really not talking about"
+    },
+    {
+      "start": 764.24,
+      "duration": 0.0,
+      "text": "and here I'm really not talking about training<00:12:44.519><c> a</c><00:12:44.639><c> large</c><00:12:44.839><c> language</c><00:12:45.160><c> model</c><00:12:45.480><c> yet</c><00:12:45.639><c> this</c>"
+    },
+    {
+      "start": 765.71,
+      "duration": 0.0,
+      "text": "training a large language model yet this"
+    },
+    {
+      "start": 765.72,
+      "duration": 0.0,
+      "text": "training a large language model yet this is<00:12:45.880><c> purely</c><00:12:46.120><c> for</c><00:12:46.279><c> the</c><00:12:46.399><c> tokenization</c><00:12:47.040><c> step</c><00:12:47.920><c> uh</c>"
+    },
+    {
+      "start": 768.03,
+      "duration": 0.0,
+      "text": "is purely for the tokenization step uh"
+    },
+    {
+      "start": 768.04,
+      "duration": 0.0,
+      "text": "is purely for the tokenization step uh so<00:12:48.240><c> this</c><00:12:48.360><c> is</c><00:12:48.519><c> my</c><00:12:48.720><c> large</c><00:12:49.199><c> Corpus</c><00:12:49.600><c> of</c><00:12:49.839><c> text</c><00:12:50.240><c> with</c>"
+    },
+    {
+      "start": 770.43,
+      "duration": 0.0,
+      "text": "so this is my large Corpus of text with"
+    },
+    {
+      "start": 770.44,
+      "duration": 0.0,
+      "text": "so this is my large Corpus of text with these<00:12:50.760><c> five</c><00:12:51.079><c> words</c><00:12:52.079><c> um</c><00:12:52.720><c> then</c><00:12:53.160><c> you</c><00:12:53.720><c> associate</c>"
+    },
+    {
+      "start": 774.389,
+      "duration": 0.0,
+      "text": "these five words um then you associate"
+    },
+    {
+      "start": 774.399,
+      "duration": 0.0,
+      "text": "these five words um then you associate every<00:12:54.760><c> character</c><00:12:55.519><c> in</c><00:12:55.720><c> this</c><00:12:55.920><c> Corpus</c><00:12:56.279><c> of</c><00:12:56.480><c> text</c><00:12:57.240><c> a</c>"
+    },
+    {
+      "start": 777.47,
+      "duration": 0.0,
+      "text": "every character in this Corpus of text a"
+    },
+    {
+      "start": 777.48,
+      "duration": 0.0,
+      "text": "every character in this Corpus of text a different<00:12:57.760><c> token</c><00:12:58.639><c> uh</c><00:12:58.760><c> so</c><00:12:58.920><c> here</c><00:12:59.199><c> I</c><00:12:59.279><c> just</c><00:12:59.399><c> split</c>"
+    },
+    {
+      "start": 779.71,
+      "duration": 0.0,
+      "text": "different token uh so here I just split"
+    },
+    {
+      "start": 779.72,
+      "duration": 0.0,
+      "text": "different token uh so here I just split up<00:12:59.880><c> every</c><00:13:00.120><c> character</c><00:13:00.480><c> with</c><00:13:00.639><c> a</c><00:13:00.880><c> different</c>"
+    },
+    {
+      "start": 781.15,
+      "duration": 0.0,
+      "text": "up every character with a different"
+    },
+    {
+      "start": 781.16,
+      "duration": 0.0,
+      "text": "up every character with a different token<00:13:01.920><c> uh</c><00:13:02.320><c> and</c><00:13:02.600><c> I</c><00:13:02.760><c> just</c><00:13:02.959><c> color</c><00:13:03.279><c> coded</c><00:13:03.680><c> all</c><00:13:03.839><c> of</c>"
+    },
+    {
+      "start": 784.03,
+      "duration": 0.0,
+      "text": "token uh and I just color coded all of"
+    },
+    {
+      "start": 784.04,
+      "duration": 0.0,
+      "text": "token uh and I just color coded all of those<00:13:04.959><c> tokens</c><00:13:05.959><c> and</c><00:13:06.079><c> then</c><00:13:06.240><c> what</c><00:13:06.320><c> you</c><00:13:06.480><c> do</c><00:13:06.800><c> is</c>"
+    },
+    {
+      "start": 786.949,
+      "duration": 0.0,
+      "text": "those tokens and then what you do is"
+    },
+    {
+      "start": 786.959,
+      "duration": 0.0,
+      "text": "those tokens and then what you do is that<00:13:07.120><c> you</c><00:13:07.279><c> go</c><00:13:07.399><c> through</c><00:13:07.600><c> your</c><00:13:07.800><c> text</c><00:13:08.160><c> and</c><00:13:08.360><c> every</c>"
+    },
+    {
+      "start": 788.59,
+      "duration": 0.0,
+      "text": "that you go through your text and every"
+    },
+    {
+      "start": 788.6,
+      "duration": 0.0,
+      "text": "that you go through your text and every time<00:13:08.800><c> you</c><00:13:08.920><c> see</c><00:13:09.720><c> pairs</c><00:13:10.040><c> of</c><00:13:10.240><c> tokens</c><00:13:10.959><c> that</c><00:13:11.120><c> are</c>"
+    },
+    {
+      "start": 791.55,
+      "duration": 0.0,
+      "text": "time you see pairs of tokens that are"
+    },
+    {
+      "start": 791.56,
+      "duration": 0.0,
+      "text": "time you see pairs of tokens that are very<00:13:11.839><c> common</c><00:13:12.440><c> the</c><00:13:12.600><c> most</c><00:13:12.920><c> common</c><00:13:13.279><c> pair</c><00:13:13.480><c> of</c>"
+    },
+    {
+      "start": 793.629,
+      "duration": 0.0,
+      "text": "very common the most common pair of"
+    },
+    {
+      "start": 793.639,
+      "duration": 0.0,
+      "text": "very common the most common pair of token<00:13:14.120><c> you</c><00:13:14.279><c> just</c><00:13:14.440><c> merge</c><00:13:14.839><c> them</c><00:13:15.240><c> so</c><00:13:15.440><c> here</c><00:13:15.560><c> you</c>"
+    },
+    {
+      "start": 795.67,
+      "duration": 0.0,
+      "text": "token you just merge them so here you"
+    },
+    {
+      "start": 795.68,
+      "duration": 0.0,
+      "text": "token you just merge them so here you see<00:13:16.000><c> three</c><00:13:16.240><c> times</c><00:13:17.000><c> the</c><00:13:17.320><c> the</c><00:13:17.920><c> the</c><00:13:18.079><c> tokens</c><00:13:18.760><c> T</c><00:13:19.120><c> and</c>"
+    },
+    {
+      "start": 799.31,
+      "duration": 0.0,
+      "text": "see three times the the the tokens T and"
+    },
+    {
+      "start": 799.32,
+      "duration": 0.0,
+      "text": "see three times the the the tokens T and O<00:13:19.880><c> next</c><00:13:20.079><c> to</c><00:13:20.240><c> each</c><00:13:20.360><c> other</c><00:13:20.639><c> so</c><00:13:20.800><c> you're</c><00:13:20.920><c> just</c>"
+    },
+    {
+      "start": 801.03,
+      "duration": 0.0,
+      "text": "O next to each other so you're just"
+    },
+    {
+      "start": 801.04,
+      "duration": 0.0,
+      "text": "O next to each other so you're just going<00:13:21.120><c> to</c><00:13:21.240><c> say</c><00:13:21.399><c> this</c><00:13:21.480><c> is</c><00:13:21.560><c> a</c><00:13:21.720><c> new</c><00:13:21.920><c> token</c><00:13:22.760><c> and</c>"
+    },
+    {
+      "start": 802.829,
+      "duration": 0.0,
+      "text": "going to say this is a new token and"
+    },
+    {
+      "start": 802.839,
+      "duration": 0.0,
+      "text": "going to say this is a new token and then<00:13:22.959><c> you</c><00:13:23.160><c> continue</c><00:13:23.519><c> you</c><00:13:23.639><c> repeat</c><00:13:24.040><c> that</c><00:13:24.360><c> so</c><00:13:24.519><c> now</c>"
+    },
+    {
+      "start": 804.67,
+      "duration": 0.0,
+      "text": "then you continue you repeat that so now"
+    },
+    {
+      "start": 804.68,
+      "duration": 0.0,
+      "text": "then you continue you repeat that so now you<00:13:24.880><c> have</c><00:13:25.399><c> to</c><00:13:26.399><c> talk</c><00:13:26.959><c> which</c><00:13:27.160><c> happens</c><00:13:27.519><c> three</c>"
+    },
+    {
+      "start": 807.79,
+      "duration": 0.0,
+      "text": "you have to talk which happens three"
+    },
+    {
+      "start": 807.8,
+      "duration": 0.0,
+      "text": "you have to talk which happens three times<00:13:28.519><c> to</c><00:13:29.320><c> with</c><00:13:29.519><c> an</c><00:13:29.680><c> E</c><00:13:30.040><c> that</c><00:13:30.199><c> happens</c><00:13:30.800><c> sorry</c>"
+    },
+    {
+      "start": 811.069,
+      "duration": 0.0,
+      "text": "times to with an E that happens sorry"
+    },
+    {
+      "start": 811.079,
+      "duration": 0.0,
+      "text": "times to with an E that happens sorry two<00:13:31.320><c> times</c><00:13:32.199><c> and</c><00:13:32.800><c> an</c><00:13:33.120><c> token</c><00:13:33.639><c> which</c><00:13:33.800><c> happens</c>"
+    },
+    {
+      "start": 814.15,
+      "duration": 0.0,
+      "text": "two times and an token which happens"
+    },
+    {
+      "start": 814.16,
+      "duration": 0.0,
+      "text": "two times and an token which happens twice<00:13:34.760><c> and</c><00:13:34.880><c> then</c><00:13:35.079><c> ex</c><00:13:35.800><c> which</c><00:13:35.959><c> also</c><00:13:36.199><c> happen</c>"
+    },
+    {
+      "start": 816.55,
+      "duration": 0.0,
+      "text": "twice and then ex which also happen"
+    },
+    {
+      "start": 816.56,
+      "duration": 0.0,
+      "text": "twice and then ex which also happen twice<00:13:37.160><c> so</c><00:13:37.360><c> this</c><00:13:37.480><c> is</c><00:13:37.680><c> that</c><00:13:38.240><c> if</c><00:13:38.440><c> you</c><00:13:38.600><c> were</c><00:13:38.880><c> to</c>"
+    },
+    {
+      "start": 819.03,
+      "duration": 0.0,
+      "text": "twice so this is that if you were to"
+    },
+    {
+      "start": 819.04,
+      "duration": 0.0,
+      "text": "twice so this is that if you were to train<00:13:39.320><c> a</c><00:13:39.440><c> tokenizer</c><00:13:40.399><c> on</c><00:13:40.639><c> this</c><00:13:40.880><c> Corpus</c><00:13:41.240><c> of</c><00:13:41.440><c> text</c>"
+    },
+    {
+      "start": 821.79,
+      "duration": 0.0,
+      "text": "train a tokenizer on this Corpus of text"
+    },
+    {
+      "start": 821.8,
+      "duration": 0.0,
+      "text": "train a tokenizer on this Corpus of text which<00:13:41.880><c> is</c><00:13:42.079><c> very</c><00:13:42.279><c> small</c><00:13:43.000><c> that's</c><00:13:43.199><c> how</c><00:13:43.360><c> you</c><00:13:43.480><c> would</c>"
+    },
+    {
+      "start": 823.87,
+      "duration": 0.0,
+      "text": "which is very small that's how you would"
+    },
+    {
+      "start": 823.88,
+      "duration": 0.0,
+      "text": "which is very small that's how you would uh<00:13:43.959><c> finish</c><00:13:44.279><c> with</c><00:13:44.440><c> a</c><00:13:44.560><c> token</c><00:13:45.079><c> with</c><00:13:45.199><c> a</c><00:13:45.360><c> pre</c><00:13:45.680><c> like</c><00:13:45.760><c> a</c>"
+    },
+    {
+      "start": 825.87,
+      "duration": 0.0,
+      "text": "uh finish with a token with a pre like a"
+    },
+    {
+      "start": 825.88,
+      "duration": 0.0,
+      "text": "uh finish with a token with a pre like a trained<00:13:46.560><c> tokenizer</c><00:13:47.560><c> uh</c><00:13:47.680><c> in</c><00:13:47.839><c> reality</c><00:13:48.240><c> you</c><00:13:48.360><c> do</c>"
+    },
+    {
+      "start": 828.509,
+      "duration": 0.0,
+      "text": "trained tokenizer uh in reality you do"
+    },
+    {
+      "start": 828.519,
+      "duration": 0.0,
+      "text": "trained tokenizer uh in reality you do it<00:13:48.680><c> on</c><00:13:49.040><c> on</c><00:13:49.240><c> much</c><00:13:49.519><c> larger</c><00:13:49.959><c> corpuses</c><00:13:50.399><c> of</c><00:13:50.600><c> text</c><00:13:51.480><c> um</c>"
+    },
+    {
+      "start": 831.829,
+      "duration": 0.0,
+      "text": "it on on much larger corpuses of text um"
+    },
+    {
+      "start": 831.839,
+      "duration": 0.0,
+      "text": "it on on much larger corpuses of text um and<00:13:52.040><c> this</c><00:13:52.160><c> is</c><00:13:52.279><c> the</c><00:13:52.480><c> real</c><00:13:53.000><c> tokenizer</c><00:13:54.000><c> of</c><00:13:54.399><c> uh</c>"
+    },
+    {
+      "start": 834.91,
+      "duration": 0.0,
+      "text": "and this is the real tokenizer of uh"
+    },
+    {
+      "start": 834.92,
+      "duration": 0.0,
+      "text": "and this is the real tokenizer of uh actually<00:13:55.240><c> I</c><00:13:55.360><c> think</c><00:13:55.560><c> this</c><00:13:55.639><c> is</c><00:13:55.800><c> gpt3</c><00:13:56.519><c> or</c><00:13:56.759><c> chat</c>"
+    },
+    {
+      "start": 836.949,
+      "duration": 0.0,
+      "text": "actually I think this is gpt3 or chat"
+    },
+    {
+      "start": 836.959,
+      "duration": 0.0,
+      "text": "actually I think this is gpt3 or chat GPT<00:13:57.920><c> uh</c><00:13:58.040><c> and</c><00:13:58.199><c> here</c><00:13:58.320><c> you</c><00:13:58.440><c> see</c><00:13:58.639><c> how</c><00:13:58.759><c> it</c><00:13:58.880><c> would</c>"
+    },
+    {
+      "start": 839.189,
+      "duration": 0.0,
+      "text": "GPT uh and here you see how it would"
+    },
+    {
+      "start": 839.199,
+      "duration": 0.0,
+      "text": "GPT uh and here you see how it would actually<00:13:59.399><c> separate</c><00:13:59.839><c> these</c><00:14:00.000><c> words</c><00:14:00.360><c> so</c>"
+    },
+    {
+      "start": 840.509,
+      "duration": 0.0,
+      "text": "actually separate these words so"
+    },
+    {
+      "start": 840.519,
+      "duration": 0.0,
+      "text": "actually separate these words so basically<00:14:00.880><c> you</c><00:14:01.000><c> see</c><00:14:01.199><c> the</c><00:14:01.320><c> same</c><00:14:01.480><c> thing</c><00:14:01.639><c> as</c><00:14:01.800><c> what</c>"
+    },
+    {
+      "start": 841.949,
+      "duration": 0.0,
+      "text": "basically you see the same thing as what"
+    },
+    {
+      "start": 841.959,
+      "duration": 0.0,
+      "text": "basically you see the same thing as what we<00:14:02.199><c> gave</c><00:14:02.560><c> in</c><00:14:02.680><c> the</c><00:14:02.839><c> previous</c><00:14:03.199><c> example</c><00:14:03.959><c> token</c>"
+    },
+    {
+      "start": 844.749,
+      "duration": 0.0,
+      "text": "we gave in the previous example token"
+    },
+    {
+      "start": 844.759,
+      "duration": 0.0,
+      "text": "we gave in the previous example token becomes<00:14:05.120><c> its</c><00:14:05.279><c> own</c><00:14:05.639><c> token</c><00:14:06.519><c> so</c><00:14:06.880><c> tokenizer</c><00:14:07.880><c> is</c>"
+    },
+    {
+      "start": 848.069,
+      "duration": 0.0,
+      "text": "becomes its own token so tokenizer is"
+    },
+    {
+      "start": 848.079,
+      "duration": 0.0,
+      "text": "becomes its own token so tokenizer is actually<00:14:08.279><c> split</c><00:14:08.639><c> up</c><00:14:08.800><c> into</c><00:14:09.040><c> two</c><00:14:09.320><c> tokens</c><00:14:10.040><c> token</c>"
+    },
+    {
+      "start": 850.67,
+      "duration": 0.0,
+      "text": "actually split up into two tokens token"
+    },
+    {
+      "start": 850.68,
+      "duration": 0.0,
+      "text": "actually split up into two tokens token and<00:14:11.079><c> iser</c><00:14:12.079><c> um</c><00:14:12.839><c> so</c><00:14:13.079><c> yeah</c><00:14:13.480><c> that's</c><00:14:13.680><c> all</c><00:14:13.880><c> about</c>"
+    },
+    {
+      "start": 854.15,
+      "duration": 0.0,
+      "text": "and iser um so yeah that's all about"
+    },
+    {
+      "start": 854.16,
+      "duration": 0.0,
+      "text": "and iser um so yeah that's all about tokenizers<00:14:15.160><c> any</c><00:14:15.320><c> questions</c><00:14:15.560><c> on</c><00:14:15.800><c> that</c><00:14:16.279><c> yeah</c>"
+    },
+    {
+      "start": 856.59,
+      "duration": 0.0,
+      "text": "tokenizers any questions on that yeah"
+    },
+    {
+      "start": 856.6,
+      "duration": 0.0,
+      "text": "tokenizers any questions on that yeah how<00:14:16.680><c> do</c><00:14:16.759><c> you</c><00:14:16.880><c> deal</c><00:14:17.040><c> with</c><00:14:17.199><c> spes</c><00:14:17.560><c> and</c><00:14:17.720><c> how</c><00:14:17.800><c> do</c><00:14:17.880><c> you</c>"
+    },
+    {
+      "start": 857.99,
+      "duration": 0.0,
+      "text": "how do you deal with spes and how do you"
+    },
+    {
+      "start": 858.0,
+      "duration": 0.0,
+      "text": "how do you deal with spes and how do you deal"
+    },
+    {
+      "start": 859.03,
+      "duration": 0.0,
+      "text": "deal"
+    },
+    {
+      "start": 859.04,
+      "duration": 0.0,
+      "text": "deal with<00:14:20.040><c> yeah</c><00:14:20.600><c> so</c><00:14:21.240><c> actually</c><00:14:21.600><c> there's</c><00:14:21.759><c> a</c><00:14:22.120><c> a</c><00:14:22.279><c> step</c>"
+    },
+    {
+      "start": 862.509,
+      "duration": 0.0,
+      "text": "with yeah so actually there's a a step"
+    },
+    {
+      "start": 862.519,
+      "duration": 0.0,
+      "text": "with yeah so actually there's a a step before<00:14:22.800><c> tokenizers</c><00:14:23.560><c> which</c><00:14:23.680><c> is</c><00:14:23.880><c> what</c><00:14:24.000><c> we</c><00:14:24.120><c> call</c>"
+    },
+    {
+      "start": 864.31,
+      "duration": 0.0,
+      "text": "before tokenizers which is what we call"
+    },
+    {
+      "start": 864.32,
+      "duration": 0.0,
+      "text": "before tokenizers which is what we call pre-<00:14:24.560><c> tokenizers</c><00:14:25.480><c> which</c><00:14:25.639><c> is</c><00:14:26.320><c> exactly</c><00:14:26.759><c> what</c>"
+    },
+    {
+      "start": 866.87,
+      "duration": 0.0,
+      "text": "pre- tokenizers which is exactly what"
+    },
+    {
+      "start": 866.88,
+      "duration": 0.0,
+      "text": "pre- tokenizers which is exactly what you<00:14:27.040><c> just</c><00:14:27.199><c> said</c><00:14:27.880><c> uh</c><00:14:27.959><c> so</c><00:14:28.160><c> this</c><00:14:28.279><c> is</c><00:14:28.519><c> mostly</c>"
+    },
+    {
+      "start": 869.59,
+      "duration": 0.0,
+      "text": "you just said uh so this is mostly"
+    },
+    {
+      "start": 869.6,
+      "duration": 0.0,
+      "text": "you just said uh so this is mostly in<00:14:29.800><c> theory</c><00:14:30.160><c> there's</c><00:14:30.360><c> no</c><00:14:30.560><c> reason</c><00:14:30.800><c> to</c><00:14:31.000><c> deal</c><00:14:31.240><c> with</c>"
+    },
+    {
+      "start": 871.509,
+      "duration": 0.0,
+      "text": "in theory there's no reason to deal with"
+    },
+    {
+      "start": 871.519,
+      "duration": 0.0,
+      "text": "in theory there's no reason to deal with spaces<00:14:32.279><c> and</c><00:14:32.720><c> punctuation</c><00:14:33.720><c> separately</c><00:14:34.240><c> you</c>"
+    },
+    {
+      "start": 874.35,
+      "duration": 0.0,
+      "text": "spaces and punctuation separately you"
+    },
+    {
+      "start": 874.36,
+      "duration": 0.0,
+      "text": "spaces and punctuation separately you could<00:14:34.519><c> just</c><00:14:34.680><c> say</c><00:14:35.000><c> every</c><00:14:35.320><c> space</c><00:14:35.680><c> gets</c><00:14:35.920><c> its</c><00:14:36.079><c> own</c>"
+    },
+    {
+      "start": 876.35,
+      "duration": 0.0,
+      "text": "could just say every space gets its own"
+    },
+    {
+      "start": 876.36,
+      "duration": 0.0,
+      "text": "could just say every space gets its own token<00:14:37.120><c> every</c><00:14:38.040><c> um</c><00:14:38.920><c> uh</c><00:14:39.040><c> punctuation</c><00:14:39.560><c> get</c><00:14:39.759><c> its</c>"
+    },
+    {
+      "start": 879.91,
+      "duration": 0.0,
+      "text": "token every um uh punctuation get its"
+    },
+    {
+      "start": 879.92,
+      "duration": 0.0,
+      "text": "token every um uh punctuation get its own<00:14:40.160><c> token</c><00:14:40.639><c> and</c><00:14:40.759><c> you</c><00:14:40.839><c> can</c><00:14:41.000><c> just</c><00:14:41.160><c> do</c><00:14:41.360><c> all</c><00:14:41.519><c> the</c>"
+    },
+    {
+      "start": 881.629,
+      "duration": 0.0,
+      "text": "own token and you can just do all the"
+    },
+    {
+      "start": 881.639,
+      "duration": 0.0,
+      "text": "own token and you can just do all the merging<00:14:42.399><c> the</c><00:14:42.600><c> problem</c><00:14:42.880><c> is</c><00:14:43.079><c> that</c><00:14:43.240><c> so</c><00:14:43.440><c> there's</c>"
+    },
+    {
+      "start": 883.55,
+      "duration": 0.0,
+      "text": "merging the problem is that so there's"
+    },
+    {
+      "start": 883.56,
+      "duration": 0.0,
+      "text": "merging the problem is that so there's an<00:14:43.720><c> efficiency</c><00:14:44.240><c> question</c><00:14:44.959><c> actually</c><00:14:45.360><c> training</c>"
+    },
+    {
+      "start": 885.67,
+      "duration": 0.0,
+      "text": "an efficiency question actually training"
+    },
+    {
+      "start": 885.68,
+      "duration": 0.0,
+      "text": "an efficiency question actually training these<00:14:45.839><c> tokenizes</c><00:14:46.480><c> takes</c><00:14:46.680><c> a</c><00:14:46.920><c> long</c><00:14:47.360><c> time</c><00:14:48.120><c> uh</c><00:14:48.240><c> so</c>"
+    },
+    {
+      "start": 888.43,
+      "duration": 0.0,
+      "text": "these tokenizes takes a long time uh so"
+    },
+    {
+      "start": 888.44,
+      "duration": 0.0,
+      "text": "these tokenizes takes a long time uh so you<00:14:48.639><c> better</c><00:14:48.959><c> off</c><00:14:49.120><c> because</c><00:14:49.279><c> you</c><00:14:49.399><c> have</c><00:14:49.519><c> to</c>"
+    },
+    {
+      "start": 889.71,
+      "duration": 0.0,
+      "text": "you better off because you have to"
+    },
+    {
+      "start": 889.72,
+      "duration": 0.0,
+      "text": "you better off because you have to consider<00:14:50.519><c> every</c><00:14:50.880><c> pair</c><00:14:51.079><c> of</c><00:14:51.240><c> token</c><00:14:51.880><c> so</c><00:14:52.040><c> what</c><00:14:52.160><c> you</c>"
+    },
+    {
+      "start": 892.269,
+      "duration": 0.0,
+      "text": "consider every pair of token so what you"
+    },
+    {
+      "start": 892.279,
+      "duration": 0.0,
+      "text": "consider every pair of token so what you end<00:14:52.480><c> up</c><00:14:52.639><c> doing</c><00:14:52.880><c> is</c><00:14:53.000><c> saying</c><00:14:53.279><c> if</c><00:14:53.399><c> there's</c><00:14:53.519><c> a</c>"
+    },
+    {
+      "start": 893.67,
+      "duration": 0.0,
+      "text": "end up doing is saying if there's a"
+    },
+    {
+      "start": 893.68,
+      "duration": 0.0,
+      "text": "end up doing is saying if there's a space<00:14:54.240><c> this</c><00:14:54.320><c> is</c><00:14:54.560><c> very</c><00:14:54.800><c> like</c><00:14:54.959><c> pre-</c><00:14:55.199><c> tokenizes</c>"
+    },
+    {
+      "start": 895.629,
+      "duration": 0.0,
+      "text": "space this is very like pre- tokenizes"
+    },
+    {
+      "start": 895.639,
+      "duration": 0.0,
+      "text": "space this is very like pre- tokenizes are<00:14:55.839><c> very</c><00:14:56.000><c> English</c><00:14:56.399><c> specific</c><00:14:57.040><c> you</c><00:14:57.199><c> say</c><00:14:57.399><c> if</c>"
+    },
+    {
+      "start": 897.509,
+      "duration": 0.0,
+      "text": "are very English specific you say if"
+    },
+    {
+      "start": 897.519,
+      "duration": 0.0,
+      "text": "are very English specific you say if there's<00:14:57.680><c> a</c><00:14:57.839><c> space</c><00:14:58.360><c> we're</c><00:14:58.519><c> not</c><00:14:58.680><c> going</c><00:14:58.759><c> to</c><00:14:59.120><c> start</c>"
+    },
+    {
+      "start": 899.35,
+      "duration": 0.0,
+      "text": "there's a space we're not going to start"
+    },
+    {
+      "start": 899.36,
+      "duration": 0.0,
+      "text": "there's a space we're not going to start looking<00:14:59.600><c> at</c><00:14:59.839><c> the</c><00:15:00.040><c> the</c><00:15:00.120><c> token</c><00:15:00.440><c> that</c><00:15:00.600><c> came</c>"
+    },
+    {
+      "start": 900.829,
+      "duration": 0.0,
+      "text": "looking at the the token that came"
+    },
+    {
+      "start": 900.839,
+      "duration": 0.0,
+      "text": "looking at the the token that came before<00:15:01.519><c> and</c><00:15:01.639><c> the</c><00:15:01.759><c> token</c><00:15:02.079><c> that</c><00:15:02.320><c> came</c>"
+    },
+    {
+      "start": 902.59,
+      "duration": 0.0,
+      "text": "before and the token that came"
+    },
+    {
+      "start": 902.6,
+      "duration": 0.0,
+      "text": "before and the token that came afterwards<00:15:03.160><c> so</c><00:15:03.279><c> you're</c><00:15:03.399><c> not</c><00:15:03.639><c> merging</c><00:15:04.240><c> in</c>"
+    },
+    {
+      "start": 904.43,
+      "duration": 0.0,
+      "text": "afterwards so you're not merging in"
+    },
+    {
+      "start": 904.44,
+      "duration": 0.0,
+      "text": "afterwards so you're not merging in between<00:15:05.320><c> spaces</c><00:15:06.079><c> but</c><00:15:06.240><c> this</c><00:15:06.360><c> is</c><00:15:06.560><c> just</c><00:15:06.759><c> like</c><00:15:07.000><c> a</c>"
+    },
+    {
+      "start": 907.47,
+      "duration": 0.0,
+      "text": "between spaces but this is just like a"
+    },
+    {
+      "start": 907.48,
+      "duration": 0.0,
+      "text": "between spaces but this is just like a optimiz<00:15:08.399><c> like</c><00:15:08.519><c> a</c><00:15:08.680><c> computation</c><00:15:09.240><c> optimization</c>"
+    },
+    {
+      "start": 910.03,
+      "duration": 0.0,
+      "text": "optimiz like a computation optimization"
+    },
+    {
+      "start": 910.04,
+      "duration": 0.0,
+      "text": "optimiz like a computation optimization you<00:15:10.199><c> could</c><00:15:10.639><c> theoretically</c><00:15:11.199><c> just</c><00:15:11.399><c> deal</c><00:15:11.639><c> with</c>"
+    },
+    {
+      "start": 911.79,
+      "duration": 0.0,
+      "text": "you could theoretically just deal with"
+    },
+    {
+      "start": 911.8,
+      "duration": 0.0,
+      "text": "you could theoretically just deal with it<00:15:12.519><c> um</c><00:15:12.759><c> the</c><00:15:12.839><c> same</c><00:15:13.040><c> way</c><00:15:13.199><c> as</c><00:15:13.279><c> you</c><00:15:13.399><c> deal</c><00:15:13.600><c> with</c><00:15:13.759><c> any</c>"
+    },
+    {
+      "start": 913.949,
+      "duration": 0.0,
+      "text": "it um the same way as you deal with any"
+    },
+    {
+      "start": 913.959,
+      "duration": 0.0,
+      "text": "it um the same way as you deal with any other<00:15:14.320><c> character</c><00:15:15.320><c> and</c><00:15:15.959><c> yeah</c><00:15:16.399><c> when</c><00:15:16.480><c> you</c><00:15:16.680><c> merge</c>"
+    },
+    {
+      "start": 917.03,
+      "duration": 0.0,
+      "text": "other character and yeah when you merge"
+    },
+    {
+      "start": 917.04,
+      "duration": 0.0,
+      "text": "other character and yeah when you merge tokens<00:15:17.440><c> do</c><00:15:17.519><c> you</c><00:15:17.839><c> delete</c><00:15:18.360><c> the</c><00:15:18.480><c> tokens</c><00:15:18.839><c> that</c><00:15:18.920><c> you</c>"
+    },
+    {
+      "start": 919.03,
+      "duration": 0.0,
+      "text": "tokens do you delete the tokens that you"
+    },
+    {
+      "start": 919.04,
+      "duration": 0.0,
+      "text": "tokens do you delete the tokens that you merged<00:15:19.440><c> away</c><00:15:19.720><c> or</c><00:15:19.959><c> do</c><00:15:20.040><c> you</c><00:15:20.279><c> keep</c><00:15:20.880><c> the</c><00:15:21.160><c> the</c>"
+    },
+    {
+      "start": 921.269,
+      "duration": 0.0,
+      "text": "merged away or do you keep the the"
+    },
+    {
+      "start": 921.279,
+      "duration": 0.0,
+      "text": "merged away or do you keep the the smaller<00:15:21.639><c> tokens</c><00:15:22.000><c> that</c><00:15:22.240><c> merge</c><00:15:22.800><c> um</c><00:15:23.160><c> you</c>"
+    },
+    {
+      "start": 923.31,
+      "duration": 0.0,
+      "text": "smaller tokens that merge um you"
+    },
+    {
+      "start": 923.32,
+      "duration": 0.0,
+      "text": "smaller tokens that merge um you actually<00:15:23.759><c> keep</c><00:15:24.040><c> the</c><00:15:24.240><c> smaller</c><00:15:24.720><c> tokens</c><00:15:25.240><c> I</c><00:15:25.320><c> mean</c>"
+    },
+    {
+      "start": 925.43,
+      "duration": 0.0,
+      "text": "actually keep the smaller tokens I mean"
+    },
+    {
+      "start": 925.44,
+      "duration": 0.0,
+      "text": "actually keep the smaller tokens I mean in<00:15:25.600><c> reality</c><00:15:25.920><c> it</c><00:15:26.000><c> doesn't</c><00:15:26.279><c> matter</c><00:15:26.560><c> much</c>"
+    },
+    {
+      "start": 926.87,
+      "duration": 0.0,
+      "text": "in reality it doesn't matter much"
+    },
+    {
+      "start": 926.88,
+      "duration": 0.0,
+      "text": "in reality it doesn't matter much because<00:15:27.959><c> um</c><00:15:29.040><c> usually</c><00:15:29.800><c> on</c><00:15:30.240><c> large</c><00:15:30.560><c> Corpus</c><00:15:30.920><c> of</c>"
+    },
+    {
+      "start": 931.03,
+      "duration": 0.0,
+      "text": "because um usually on large Corpus of"
+    },
+    {
+      "start": 931.04,
+      "duration": 0.0,
+      "text": "because um usually on large Corpus of text<00:15:31.279><c> you</c><00:15:31.360><c> will</c><00:15:31.519><c> have</c><00:15:31.720><c> actually</c><00:15:31.959><c> everything</c>"
+    },
+    {
+      "start": 932.949,
+      "duration": 0.0,
+      "text": "text you will have actually everything"
+    },
+    {
+      "start": 932.959,
+      "duration": 0.0,
+      "text": "text you will have actually everything uh<00:15:33.120><c> but</c><00:15:33.240><c> you</c><00:15:33.399><c> usually</c><00:15:33.680><c> keep</c><00:15:33.920><c> the</c><00:15:34.040><c> small</c><00:15:34.240><c> ones</c>"
+    },
+    {
+      "start": 934.43,
+      "duration": 0.0,
+      "text": "uh but you usually keep the small ones"
+    },
+    {
+      "start": 934.44,
+      "duration": 0.0,
+      "text": "uh but you usually keep the small ones and<00:15:34.560><c> the</c><00:15:34.639><c> reason</c><00:15:34.880><c> why</c><00:15:34.959><c> you</c><00:15:35.040><c> want</c><00:15:35.160><c> to</c><00:15:35.279><c> do</c><00:15:35.440><c> that</c>"
+    },
+    {
+      "start": 935.59,
+      "duration": 0.0,
+      "text": "and the reason why you want to do that"
+    },
+    {
+      "start": 935.6,
+      "duration": 0.0,
+      "text": "and the reason why you want to do that is<00:15:35.759><c> because</c><00:15:36.000><c> if</c><00:15:36.240><c> in</c><00:15:36.480><c> case</c><00:15:36.720><c> there's</c><00:15:37.240><c> as</c><00:15:37.360><c> we</c><00:15:37.519><c> said</c>"
+    },
+    {
+      "start": 937.79,
+      "duration": 0.0,
+      "text": "is because if in case there's as we said"
+    },
+    {
+      "start": 937.8,
+      "duration": 0.0,
+      "text": "is because if in case there's as we said before<00:15:38.440><c> you</c><00:15:38.680><c> have</c><00:15:38.920><c> some</c><00:15:39.360><c> um</c><00:15:39.839><c> some</c><00:15:40.040><c> grammatical</c>"
+    },
+    {
+      "start": 940.509,
+      "duration": 0.0,
+      "text": "before you have some um some grammatical"
+    },
+    {
+      "start": 940.519,
+      "duration": 0.0,
+      "text": "before you have some um some grammatical mistakes<00:15:40.839><c> so</c><00:15:41.000><c> some</c><00:15:41.120><c> typos</c><00:15:41.720><c> you</c><00:15:41.839><c> still</c><00:15:42.040><c> want</c><00:15:42.120><c> to</c>"
+    },
+    {
+      "start": 942.269,
+      "duration": 0.0,
+      "text": "mistakes so some typos you still want to"
+    },
+    {
+      "start": 942.279,
+      "duration": 0.0,
+      "text": "mistakes so some typos you still want to be<00:15:42.399><c> able</c><00:15:42.600><c> to</c><00:15:42.800><c> represent</c><00:15:43.480><c> these</c><00:15:43.680><c> words</c><00:15:44.000><c> by</c>"
+    },
+    {
+      "start": 944.59,
+      "duration": 0.0,
+      "text": "be able to represent these words by"
+    },
+    {
+      "start": 944.6,
+      "duration": 0.0,
+      "text": "be able to represent these words by character<00:15:45.600><c> um</c><00:15:46.560><c> so</c><00:15:47.040><c> yeah</c><00:15:48.040><c> yes</c><00:15:48.839><c> are</c><00:15:49.000><c> the</c><00:15:49.399><c> tokens</c>"
+    },
+    {
+      "start": 950.389,
+      "duration": 0.0,
+      "text": "character um so yeah yes are the tokens"
+    },
+    {
+      "start": 950.399,
+      "duration": 0.0,
+      "text": "character um so yeah yes are the tokens unique<00:15:51.199><c> so</c><00:15:51.600><c> I</c><00:15:51.680><c> mean</c><00:15:52.360><c> say</c><00:15:52.600><c> in</c><00:15:52.720><c> this</c><00:15:52.880><c> case</c><00:15:53.160><c> T</c><00:15:53.560><c> Ken</c>"
+    },
+    {
+      "start": 954.23,
+      "duration": 0.0,
+      "text": "unique so I mean say in this case T Ken"
+    },
+    {
+      "start": 954.24,
+      "duration": 0.0,
+      "text": "unique so I mean say in this case T Ken is<00:15:54.360><c> there</c><00:15:54.519><c> only</c><00:15:54.759><c> one</c><00:15:55.000><c> occurrence</c><00:15:55.399><c> or</c><00:15:55.720><c> could</c><00:15:56.240><c> do</c>"
+    },
+    {
+      "start": 956.43,
+      "duration": 0.0,
+      "text": "is there only one occurrence or could do"
+    },
+    {
+      "start": 956.44,
+      "duration": 0.0,
+      "text": "is there only one occurrence or could do you<00:15:56.600><c> need</c><00:15:56.800><c> to</c><00:15:57.800><c> leave</c><00:15:58.199><c> multiple</c><00:15:58.600><c> occurr</c><00:15:59.160><c> so</c>"
+    },
+    {
+      "start": 959.309,
+      "duration": 0.0,
+      "text": "you need to leave multiple occurr so"
+    },
+    {
+      "start": 959.319,
+      "duration": 0.0,
+      "text": "you need to leave multiple occurr so they<00:15:59.440><c> could</c><00:15:59.639><c> have</c><00:16:00.160><c> take</c><00:16:00.319><c> on</c><00:16:00.560><c> different</c>"
+    },
+    {
+      "start": 960.829,
+      "duration": 0.0,
+      "text": "they could have take on different"
+    },
+    {
+      "start": 960.839,
+      "duration": 0.0,
+      "text": "they could have take on different meanings<00:16:01.240><c> or</c><00:16:01.399><c> something</c><00:16:01.959><c> oh</c><00:16:02.199><c> oh</c><00:16:02.360><c> I</c><00:16:02.440><c> see</c><00:16:02.600><c> what</c>"
+    },
+    {
+      "start": 962.67,
+      "duration": 0.0,
+      "text": "meanings or something oh oh I see what"
+    },
+    {
+      "start": 962.68,
+      "duration": 0.0,
+      "text": "meanings or something oh oh I see what you<00:16:02.959><c> say</c><00:16:03.199><c> no</c><00:16:03.480><c> no</c><00:16:03.639><c> it's</c><00:16:03.959><c> every</c><00:16:04.240><c> token</c><00:16:04.880><c> has</c><00:16:05.079><c> its</c>"
+    },
+    {
+      "start": 965.309,
+      "duration": 0.0,
+      "text": "you say no no it's every token has its"
+    },
+    {
+      "start": 965.319,
+      "duration": 0.0,
+      "text": "you say no no it's every token has its own<00:16:06.319><c> uh</c><00:16:06.600><c> unique</c><00:16:07.240><c> ID</c><00:16:08.240><c> um</c><00:16:08.759><c> so</c><00:16:09.199><c> a</c><00:16:09.399><c> usual</c><00:16:10.160><c> this</c><00:16:10.240><c> is</c><00:16:10.319><c> a</c>"
+    },
+    {
+      "start": 970.47,
+      "duration": 0.0,
+      "text": "own uh unique ID um so a usual this is a"
+    },
+    {
+      "start": 970.48,
+      "duration": 0.0,
+      "text": "own uh unique ID um so a usual this is a great<00:16:10.680><c> question</c><00:16:10.959><c> for</c><00:16:11.120><c> example</c><00:16:11.399><c> if</c><00:16:11.480><c> you</c><00:16:11.600><c> think</c>"
+    },
+    {
+      "start": 971.79,
+      "duration": 0.0,
+      "text": "great question for example if you think"
+    },
+    {
+      "start": 971.8,
+      "duration": 0.0,
+      "text": "great question for example if you think about<00:16:12.319><c> a</c><00:16:12.680><c> bank</c><00:16:13.199><c> which</c><00:16:13.360><c> could</c><00:16:13.480><c> be</c><00:16:13.639><c> bank</c><00:16:13.880><c> for</c>"
+    },
+    {
+      "start": 974.03,
+      "duration": 0.0,
+      "text": "about a bank which could be bank for"
+    },
+    {
+      "start": 974.04,
+      "duration": 0.0,
+      "text": "about a bank which could be bank for like<00:16:14.199><c> money</c><00:16:14.440><c> or</c><00:16:14.600><c> bank</c><00:16:14.880><c> like</c><00:16:15.120><c> water</c><00:16:16.040><c> um</c><00:16:16.440><c> it</c><00:16:16.639><c> will</c>"
+    },
+    {
+      "start": 976.87,
+      "duration": 0.0,
+      "text": "like money or bank like water um it will"
+    },
+    {
+      "start": 976.88,
+      "duration": 0.0,
+      "text": "like money or bank like water um it will have<00:16:17.040><c> the</c><00:16:17.199><c> same</c><00:16:17.440><c> token</c><00:16:18.120><c> but</c><00:16:18.279><c> the</c><00:16:18.440><c> model</c><00:16:18.800><c> will</c>"
+    },
+    {
+      "start": 978.949,
+      "duration": 0.0,
+      "text": "have the same token but the model will"
+    },
+    {
+      "start": 978.959,
+      "duration": 0.0,
+      "text": "have the same token but the model will learn<00:16:19.279><c> the</c><00:16:19.399><c> Transformer</c><00:16:19.959><c> will</c><00:16:20.160><c> learn</c><00:16:20.639><c> that</c>"
+    },
+    {
+      "start": 980.87,
+      "duration": 0.0,
+      "text": "learn the Transformer will learn that"
+    },
+    {
+      "start": 980.88,
+      "duration": 0.0,
+      "text": "learn the Transformer will learn that based<00:16:21.160><c> on</c><00:16:21.279><c> the</c><00:16:21.399><c> words</c><00:16:21.680><c> that</c><00:16:21.800><c> are</c><00:16:21.959><c> around</c><00:16:22.240><c> it</c><00:16:22.880><c> it</c>"
+    },
+    {
+      "start": 983.189,
+      "duration": 0.0,
+      "text": "based on the words that are around it it"
+    },
+    {
+      "start": 983.199,
+      "duration": 0.0,
+      "text": "based on the words that are around it it should<00:16:23.519><c> associate</c><00:16:24.199><c> that</c><00:16:24.959><c> I'm</c><00:16:25.079><c> saying</c><00:16:25.279><c> I'm</c>"
+    },
+    {
+      "start": 985.35,
+      "duration": 0.0,
+      "text": "should associate that I'm saying I'm"
+    },
+    {
+      "start": 985.36,
+      "duration": 0.0,
+      "text": "should associate that I'm saying I'm being<00:16:25.600><c> very</c><00:16:25.800><c> high</c><00:16:26.040><c> wavy</c><00:16:26.399><c> here</c><00:16:26.560><c> but</c><00:16:26.800><c> associate</c>"
+    },
+    {
+      "start": 987.269,
+      "duration": 0.0,
+      "text": "being very high wavy here but associate"
+    },
+    {
+      "start": 987.279,
+      "duration": 0.0,
+      "text": "being very high wavy here but associate that<00:16:27.440><c> with</c><00:16:27.639><c> the</c><00:16:28.000><c> with</c><00:16:28.120><c> a</c><00:16:28.560><c> with</c><00:16:28.720><c> a</c>"
+    },
+    {
+      "start": 989.269,
+      "duration": 0.0,
+      "text": "that with the with a with a"
+    },
+    {
+      "start": 989.279,
+      "duration": 0.0,
+      "text": "that with the with a with a representation<00:16:30.279><c> that</c><00:16:30.440><c> is</c><00:16:30.639><c> either</c><00:16:30.920><c> more</c><00:16:31.199><c> like</c>"
+    },
+    {
+      "start": 991.629,
+      "duration": 0.0,
+      "text": "representation that is either more like"
+    },
+    {
+      "start": 991.639,
+      "duration": 0.0,
+      "text": "representation that is either more like the<00:16:31.800><c> bank</c><00:16:32.279><c> money</c><00:16:32.639><c> side</c><00:16:32.880><c> or</c><00:16:33.040><c> the</c><00:16:33.199><c> Bank</c><00:16:33.680><c> water</c>"
+    },
+    {
+      "start": 994.069,
+      "duration": 0.0,
+      "text": "the bank money side or the Bank water"
+    },
+    {
+      "start": 994.079,
+      "duration": 0.0,
+      "text": "the bank money side or the Bank water side<00:16:34.759><c> um</c><00:16:34.920><c> but</c><00:16:35.040><c> that's</c><00:16:35.160><c> a</c><00:16:35.279><c> Transformer</c><00:16:35.759><c> that</c>"
+    },
+    {
+      "start": 995.91,
+      "duration": 0.0,
+      "text": "side um but that's a Transformer that"
+    },
+    {
+      "start": 995.92,
+      "duration": 0.0,
+      "text": "side um but that's a Transformer that does<00:16:36.160><c> that</c><00:16:36.360><c> it's</c><00:16:36.440><c> not</c><00:16:36.600><c> a</c>"
+    },
+    {
+      "start": 997.309,
+      "duration": 0.0,
+      "text": "does that it's not a"
+    },
+    {
+      "start": 997.319,
+      "duration": 0.0,
+      "text": "does that it's not a tokenizer<00:16:38.319><c> yes</c><00:16:39.279><c> yeah</c><00:16:39.399><c> so</c><00:16:39.519><c> you</c><00:16:39.639><c> mentioned</c>"
+    },
+    {
+      "start": 999.99,
+      "duration": 0.0,
+      "text": "tokenizer yes yeah so you mentioned"
+    },
+    {
+      "start": 1000.0,
+      "duration": 0.0,
+      "text": "tokenizer yes yeah so you mentioned during<00:16:40.279><c> tokenization</c><00:16:41.040><c> keep</c><00:16:41.240><c> the</c><00:16:41.360><c> smaller</c>"
+    },
+    {
+      "start": 1001.71,
+      "duration": 0.0,
+      "text": "during tokenization keep the smaller"
+    },
+    {
+      "start": 1001.72,
+      "duration": 0.0,
+      "text": "during tokenization keep the smaller tokens<00:16:42.120><c> you</c><00:16:42.240><c> started</c><00:16:42.600><c> with</c><00:16:42.880><c> right</c><00:16:43.600><c> like</c><00:16:44.199><c> if</c>"
+    },
+    {
+      "start": 1004.269,
+      "duration": 0.0,
+      "text": "tokens you started with right like if"
+    },
+    {
+      "start": 1004.279,
+      "duration": 0.0,
+      "text": "tokens you started with right like if you<00:16:44.399><c> start</c><00:16:44.600><c> with</c><00:16:44.720><c> a</c><00:16:44.920><c> t</c><00:16:45.279><c> you</c><00:16:45.440><c> keep</c><00:16:45.639><c> the</c><00:16:45.800><c> T</c><00:16:46.120><c> and</c>"
+    },
+    {
+      "start": 1006.309,
+      "duration": 0.0,
+      "text": "you start with a t you keep the T and"
+    },
+    {
+      "start": 1006.319,
+      "duration": 0.0,
+      "text": "you start with a t you keep the T and then<00:16:46.440><c> you</c><00:16:46.680><c> build</c><00:16:46.920><c> your</c><00:16:47.040><c> tokenizer</c><00:16:47.680><c> to</c><00:16:47.800><c> the</c>"
+    },
+    {
+      "start": 1008.15,
+      "duration": 0.0,
+      "text": "then you build your tokenizer to the"
+    },
+    {
+      "start": 1008.16,
+      "duration": 0.0,
+      "text": "then you build your tokenizer to the that<00:16:48.240><c> you</c><00:16:48.360><c> can</c><00:16:48.519><c> now</c><00:16:48.680><c> in</c><00:16:49.120><c> token</c><00:16:49.839><c> so</c><00:16:50.079><c> let's</c><00:16:50.279><c> say</c>"
+    },
+    {
+      "start": 1010.949,
+      "duration": 0.0,
+      "text": "that you can now in token so let's say"
+    },
+    {
+      "start": 1010.959,
+      "duration": 0.0,
+      "text": "that you can now in token so let's say maybe<00:16:51.120><c> you</c><00:16:51.240><c> didn't</c><00:16:51.399><c> train</c><00:16:51.639><c> on</c><00:16:51.839><c> token</c><00:16:52.160><c> but</c><00:16:52.319><c> like</c>"
+    },
+    {
+      "start": 1012.47,
+      "duration": 0.0,
+      "text": "maybe you didn't train on token but like"
+    },
+    {
+      "start": 1012.48,
+      "duration": 0.0,
+      "text": "maybe you didn't train on token but like in<00:16:52.600><c> your</c><00:16:52.800><c> data</c><00:16:53.160><c> you</c><00:16:53.240><c> are</c><00:16:53.360><c> trying</c><00:16:53.600><c> to</c><00:16:53.880><c> encode</c>"
+    },
+    {
+      "start": 1014.47,
+      "duration": 0.0,
+      "text": "in your data you are trying to encode"
+    },
+    {
+      "start": 1014.48,
+      "duration": 0.0,
+      "text": "in your data you are trying to encode token<00:16:55.279><c> so</c><00:16:55.519><c> how</c><00:16:55.680><c> does</c><00:16:56.000><c> the</c><00:16:56.240><c> tokenizer</c><00:16:56.880><c> know</c><00:16:57.120><c> to</c>"
+    },
+    {
+      "start": 1017.35,
+      "duration": 0.0,
+      "text": "token so how does the tokenizer know to"
+    },
+    {
+      "start": 1017.36,
+      "duration": 0.0,
+      "text": "token so how does the tokenizer know to encode<00:16:57.759><c> it</c><00:16:58.000><c> with</c><00:16:58.279><c> token</c><00:16:58.600><c> or</c>"
+    },
+    {
+      "start": 1019.99,
+      "duration": 0.0,
+      "text": "encode it with token or"
+    },
+    {
+      "start": 1020.0,
+      "duration": 0.0,
+      "text": "encode it with token or a<00:17:00.160><c> great</c><00:17:00.360><c> question</c><00:17:00.720><c> you</c><00:17:00.920><c> basically</c><00:17:01.360><c> when</c><00:17:01.519><c> you</c>"
+    },
+    {
+      "start": 1021.91,
+      "duration": 0.0,
+      "text": "a great question you basically when you"
+    },
+    {
+      "start": 1021.92,
+      "duration": 0.0,
+      "text": "a great question you basically when you so<00:17:02.199><c> when</c><00:17:02.279><c> you</c><00:17:02.440><c> tokenize</c><00:17:02.959><c> so</c><00:17:03.120><c> that's</c><00:17:03.360><c> after</c>"
+    },
+    {
+      "start": 1023.59,
+      "duration": 0.0,
+      "text": "so when you tokenize so that's after"
+    },
+    {
+      "start": 1023.6,
+      "duration": 0.0,
+      "text": "so when you tokenize so that's after training<00:17:03.920><c> of</c><00:17:04.039><c> the</c><00:17:04.160><c> tokenizer</c><00:17:04.679><c> when</c><00:17:04.760><c> you</c>"
+    },
+    {
+      "start": 1024.949,
+      "duration": 0.0,
+      "text": "training of the tokenizer when you"
+    },
+    {
+      "start": 1024.959,
+      "duration": 0.0,
+      "text": "training of the tokenizer when you actually<00:17:05.400><c> apply</c><00:17:05.720><c> the</c><00:17:05.880><c> tokenizer</c><00:17:06.640><c> you</c>"
+    },
+    {
+      "start": 1026.87,
+      "duration": 0.0,
+      "text": "actually apply the tokenizer you"
+    },
+    {
+      "start": 1026.88,
+      "duration": 0.0,
+      "text": "actually apply the tokenizer you basically<00:17:07.360><c> always</c><00:17:07.720><c> choose</c><00:17:08.079><c> the</c><00:17:08.559><c> largest</c><00:17:09.559><c> uh</c>"
+    },
+    {
+      "start": 1029.71,
+      "duration": 0.0,
+      "text": "basically always choose the largest uh"
+    },
+    {
+      "start": 1029.72,
+      "duration": 0.0,
+      "text": "basically always choose the largest uh token<00:17:10.160><c> that</c><00:17:10.240><c> you</c><00:17:10.360><c> can</c><00:17:10.520><c> apply</c><00:17:11.480><c> uh</c><00:17:11.600><c> so</c><00:17:11.760><c> if</c><00:17:11.839><c> you</c>"
+    },
+    {
+      "start": 1031.909,
+      "duration": 0.0,
+      "text": "token that you can apply uh so if you"
+    },
+    {
+      "start": 1031.919,
+      "duration": 0.0,
+      "text": "token that you can apply uh so if you can<00:17:12.079><c> do</c><00:17:12.240><c> token</c><00:17:12.559><c> you</c><00:17:12.640><c> will</c><00:17:12.799><c> never</c><00:17:13.000><c> do</c><00:17:13.160><c> T</c><00:17:13.640><c> you</c>"
+    },
+    {
+      "start": 1033.71,
+      "duration": 0.0,
+      "text": "can do token you will never do T you"
+    },
+    {
+      "start": 1033.72,
+      "duration": 0.0,
+      "text": "can do token you will never do T you will<00:17:13.919><c> always</c><00:17:14.120><c> do</c><00:17:14.559><c> token</c><00:17:15.559><c> um</c><00:17:16.199><c> but</c><00:17:16.360><c> there's</c>"
+    },
+    {
+      "start": 1036.59,
+      "duration": 0.0,
+      "text": "will always do token um but there's"
+    },
+    {
+      "start": 1036.6,
+      "duration": 0.0,
+      "text": "will always do token um but there's actually<00:17:17.000><c> so</c><00:17:17.520><c> people</c><00:17:17.760><c> don't</c><00:17:17.959><c> usually</c><00:17:18.240><c> talk</c>"
+    },
+    {
+      "start": 1038.47,
+      "duration": 0.0,
+      "text": "actually so people don't usually talk"
+    },
+    {
+      "start": 1038.48,
+      "duration": 0.0,
+      "text": "actually so people don't usually talk that<00:17:18.600><c> much</c><00:17:18.760><c> about</c><00:17:18.959><c> tokenizers</c><00:17:19.720><c> but</c><00:17:20.039><c> uh</c>"
+    },
+    {
+      "start": 1040.429,
+      "duration": 0.0,
+      "text": "that much about tokenizers but uh"
+    },
+    {
+      "start": 1040.439,
+      "duration": 0.0,
+      "text": "that much about tokenizers but uh there's<00:17:20.600><c> a</c><00:17:20.720><c> lot</c><00:17:20.880><c> of</c><00:17:21.480><c> of</c><00:17:21.760><c> computational</c>"
+    },
+    {
+      "start": 1042.35,
+      "duration": 0.0,
+      "text": "there's a lot of of computational"
+    },
+    {
+      "start": 1042.36,
+      "duration": 0.0,
+      "text": "there's a lot of of computational benefits<00:17:23.280><c> uh</c><00:17:23.480><c> or</c><00:17:23.679><c> computational</c><00:17:24.240><c> tricks</c><00:17:24.559><c> that</c>"
+    },
+    {
+      "start": 1044.63,
+      "duration": 0.0,
+      "text": "benefits uh or computational tricks that"
+    },
+    {
+      "start": 1044.64,
+      "duration": 0.0,
+      "text": "benefits uh or computational tricks that you<00:17:24.760><c> can</c><00:17:24.919><c> do</c><00:17:25.240><c> for</c><00:17:25.439><c> making</c><00:17:25.720><c> these</c><00:17:25.880><c> things</c>"
+    },
+    {
+      "start": 1046.27,
+      "duration": 0.0,
+      "text": "you can do for making these things"
+    },
+    {
+      "start": 1046.28,
+      "duration": 0.0,
+      "text": "you can do for making these things faster<00:17:27.160><c> uh</c><00:17:27.240><c> so</c><00:17:27.400><c> I</c><00:17:27.520><c> really</c><00:17:27.679><c> don't</c><00:17:27.880><c> think</c><00:17:28.079><c> we</c><00:17:28.280><c> and</c>"
+    },
+    {
+      "start": 1048.51,
+      "duration": 0.0,
+      "text": "faster uh so I really don't think we and"
+    },
+    {
+      "start": 1048.52,
+      "duration": 0.0,
+      "text": "faster uh so I really don't think we and honestly<00:17:29.080><c> I</c><00:17:29.160><c> think</c><00:17:29.280><c> a</c><00:17:29.360><c> lot</c><00:17:29.440><c> of</c><00:17:29.559><c> people</c><00:17:29.760><c> think</c>"
+    },
+    {
+      "start": 1049.95,
+      "duration": 0.0,
+      "text": "honestly I think a lot of people think"
+    },
+    {
+      "start": 1049.96,
+      "duration": 0.0,
+      "text": "honestly I think a lot of people think that<00:17:30.039><c> we</c><00:17:30.160><c> should</c><00:17:30.360><c> just</c><00:17:30.559><c> get</c><00:17:30.880><c> away</c><00:17:31.120><c> from</c>"
+    },
+    {
+      "start": 1051.669,
+      "duration": 0.0,
+      "text": "that we should just get away from"
+    },
+    {
+      "start": 1051.679,
+      "duration": 0.0,
+      "text": "that we should just get away from tokenizers<00:17:32.679><c> um</c><00:17:33.120><c> and</c><00:17:33.280><c> just</c><00:17:33.440><c> kind</c><00:17:33.559><c> of</c><00:17:33.679><c> tokenize</c>"
+    },
+    {
+      "start": 1054.19,
+      "duration": 0.0,
+      "text": "tokenizers um and just kind of tokenize"
+    },
+    {
+      "start": 1054.2,
+      "duration": 0.0,
+      "text": "tokenizers um and just kind of tokenize character<00:17:34.559><c> by</c><00:17:34.720><c> character</c><00:17:35.480><c> or</c><00:17:35.679><c> bites</c><00:17:36.000><c> by</c><00:17:36.200><c> bites</c>"
+    },
+    {
+      "start": 1056.99,
+      "duration": 0.0,
+      "text": "character by character or bites by bites"
+    },
+    {
+      "start": 1057.0,
+      "duration": 0.0,
+      "text": "character by character or bites by bites uh<00:17:37.160><c> but</c><00:17:37.280><c> as</c><00:17:37.400><c> I</c><00:17:37.520><c> said</c><00:17:37.760><c> right</c><00:17:37.880><c> now</c><00:17:38.039><c> there's</c><00:17:38.240><c> this</c>"
+    },
+    {
+      "start": 1058.43,
+      "duration": 0.0,
+      "text": "uh but as I said right now there's this"
+    },
+    {
+      "start": 1058.44,
+      "duration": 0.0,
+      "text": "uh but as I said right now there's this issue<00:17:38.679><c> of</c><00:17:38.840><c> like</c><00:17:39.000><c> length</c><00:17:39.799><c> uh</c><00:17:39.919><c> but</c><00:17:40.080><c> maybe</c><00:17:40.360><c> one</c>"
+    },
+    {
+      "start": 1060.51,
+      "duration": 0.0,
+      "text": "issue of like length uh but maybe one"
+    },
+    {
+      "start": 1060.52,
+      "duration": 0.0,
+      "text": "issue of like length uh but maybe one day<00:17:40.760><c> like</c><00:17:40.880><c> in</c><00:17:41.039><c> five</c><00:17:41.240><c> or</c><00:17:41.400><c> 10</c><00:17:41.640><c> years</c><00:17:42.280><c> we</c><00:17:42.360><c> will</c>"
+    },
+    {
+      "start": 1062.51,
+      "duration": 0.0,
+      "text": "day like in five or 10 years we will"
+    },
+    {
+      "start": 1062.52,
+      "duration": 0.0,
+      "text": "day like in five or 10 years we will have<00:17:42.679><c> different</c><00:17:42.960><c> architectures</c><00:17:43.440><c> that</c><00:17:43.559><c> don't</c>"
+    },
+    {
+      "start": 1063.71,
+      "duration": 0.0,
+      "text": "have different architectures that don't"
+    },
+    {
+      "start": 1063.72,
+      "duration": 0.0,
+      "text": "have different architectures that don't scale<00:17:44.039><c> quadratically</c><00:17:44.600><c> with</c><00:17:44.760><c> the</c><00:17:45.039><c> length</c><00:17:45.280><c> of</c>"
+    },
+    {
+      "start": 1065.39,
+      "duration": 0.0,
+      "text": "scale quadratically with the length of"
+    },
+    {
+      "start": 1065.4,
+      "duration": 0.0,
+      "text": "scale quadratically with the length of the<00:17:45.520><c> sequence</c><00:17:46.120><c> and</c><00:17:46.440><c> uh</c><00:17:46.600><c> maybe</c><00:17:47.240><c> we'll</c><00:17:48.240><c> um</c><00:17:49.080><c> yeah</c>"
+    },
+    {
+      "start": 1069.31,
+      "duration": 0.0,
+      "text": "the sequence and uh maybe we'll um yeah"
+    },
+    {
+      "start": 1069.32,
+      "duration": 0.0,
+      "text": "the sequence and uh maybe we'll um yeah move<00:17:49.600><c> away</c><00:17:49.760><c> from</c><00:17:50.000><c> tokenizes</c><00:17:51.000><c> so</c><00:17:51.280><c> can</c><00:17:51.400><c> you</c>"
+    },
+    {
+      "start": 1071.549,
+      "duration": 0.0,
+      "text": "move away from tokenizes so can you"
+    },
+    {
+      "start": 1071.559,
+      "duration": 0.0,
+      "text": "move away from tokenizes so can you share<00:17:51.840><c> with</c><00:17:51.960><c> us</c><00:17:52.120><c> the</c><00:17:52.280><c> drawback</c><00:17:53.240><c> why</c><00:17:53.360><c> do</c><00:17:53.520><c> people</c>"
+    },
+    {
+      "start": 1073.71,
+      "duration": 0.0,
+      "text": "share with us the drawback why do people"
+    },
+    {
+      "start": 1073.72,
+      "duration": 0.0,
+      "text": "share with us the drawback why do people want<00:17:53.799><c> to</c><00:17:53.960><c> move</c><00:17:54.160><c> away</c><00:17:54.360><c> from</c><00:17:54.480><c> the</c><00:17:54.679><c> tokenizer</c><00:17:55.679><c> oh</c>"
+    },
+    {
+      "start": 1076.75,
+      "duration": 0.0,
+      "text": "want to move away from the tokenizer oh"
+    },
+    {
+      "start": 1076.76,
+      "duration": 0.0,
+      "text": "want to move away from the tokenizer oh um<00:17:57.760><c> yeah</c><00:17:58.000><c> so</c><00:17:58.320><c> think</c>"
+    },
+    {
+      "start": 1080.029,
+      "duration": 0.0,
+      "text": "um yeah so think"
+    },
+    {
+      "start": 1080.039,
+      "duration": 0.0,
+      "text": "um yeah so think one<00:18:00.240><c> good</c><00:18:00.480><c> example</c><00:18:01.360><c> is</c><00:18:02.240><c> uh</c><00:18:02.640><c> math</c><00:18:03.559><c> if</c><00:18:03.640><c> you</c><00:18:03.799><c> think</c>"
+    },
+    {
+      "start": 1083.95,
+      "duration": 0.0,
+      "text": "one good example is uh math if you think"
+    },
+    {
+      "start": 1083.96,
+      "duration": 0.0,
+      "text": "one good example is uh math if you think about<00:18:04.240><c> math</c><00:18:04.760><c> actually</c><00:18:05.200><c> numbers</c><00:18:05.640><c> right</c><00:18:05.799><c> now</c>"
+    },
+    {
+      "start": 1085.99,
+      "duration": 0.0,
+      "text": "about math actually numbers right now"
+    },
+    {
+      "start": 1086.0,
+      "duration": 0.0,
+      "text": "about math actually numbers right now are<00:18:06.159><c> not</c><00:18:06.320><c> tokenized</c><00:18:07.159><c> so</c><00:18:07.320><c> for</c><00:18:07.440><c> example</c><00:18:07.840><c> 327</c>"
+    },
+    {
+      "start": 1088.83,
+      "duration": 0.0,
+      "text": "are not tokenized so for example 327"
+    },
+    {
+      "start": 1088.84,
+      "duration": 0.0,
+      "text": "are not tokenized so for example 327 might<00:18:09.039><c> have</c><00:18:09.240><c> its</c><00:18:09.360><c> own</c><00:18:09.600><c> token</c><00:18:10.400><c> which</c><00:18:10.559><c> means</c>"
+    },
+    {
+      "start": 1090.789,
+      "duration": 0.0,
+      "text": "might have its own token which means"
+    },
+    {
+      "start": 1090.799,
+      "duration": 0.0,
+      "text": "might have its own token which means that<00:18:11.000><c> models</c><00:18:11.559><c> when</c><00:18:11.760><c> they</c><00:18:11.960><c> see</c><00:18:12.440><c> numbers</c><00:18:13.120><c> they</c>"
+    },
+    {
+      "start": 1093.27,
+      "duration": 0.0,
+      "text": "that models when they see numbers they"
+    },
+    {
+      "start": 1093.28,
+      "duration": 0.0,
+      "text": "that models when they see numbers they don't<00:18:13.480><c> see</c><00:18:13.720><c> them</c><00:18:13.960><c> the</c><00:18:14.200><c> same</c><00:18:14.400><c> way</c><00:18:14.559><c> as</c><00:18:14.720><c> we</c><00:18:14.919><c> do</c><00:18:15.559><c> and</c>"
+    },
+    {
+      "start": 1095.789,
+      "duration": 0.0,
+      "text": "don't see them the same way as we do and"
+    },
+    {
+      "start": 1095.799,
+      "duration": 0.0,
+      "text": "don't see them the same way as we do and this<00:18:15.919><c> is</c><00:18:16.039><c> very</c><00:18:16.280><c> annoying</c><00:18:16.679><c> because</c><00:18:16.960><c> what</c><00:18:17.200><c> I</c>"
+    },
+    {
+      "start": 1097.23,
+      "duration": 0.0,
+      "text": "this is very annoying because what I"
+    },
+    {
+      "start": 1097.24,
+      "duration": 0.0,
+      "text": "this is very annoying because what I mean<00:18:17.480><c> the</c><00:18:17.600><c> reason</c><00:18:17.960><c> why</c><00:18:18.080><c> we</c><00:18:18.240><c> can</c><00:18:18.520><c> kind</c><00:18:18.640><c> of</c>"
+    },
+    {
+      "start": 1098.75,
+      "duration": 0.0,
+      "text": "mean the reason why we can kind of"
+    },
+    {
+      "start": 1098.76,
+      "duration": 0.0,
+      "text": "mean the reason why we can kind of generalize<00:18:19.240><c> with</c><00:18:19.400><c> math</c><00:18:19.960><c> is</c><00:18:20.120><c> because</c><00:18:20.320><c> we</c><00:18:20.440><c> can</c>"
+    },
+    {
+      "start": 1100.59,
+      "duration": 0.0,
+      "text": "generalize with math is because we can"
+    },
+    {
+      "start": 1100.6,
+      "duration": 0.0,
+      "text": "generalize with math is because we can deal<00:18:20.840><c> with</c><00:18:21.039><c> every</c><00:18:21.480><c> every</c><00:18:21.720><c> letter</c><00:18:22.080><c> separately</c>"
+    },
+    {
+      "start": 1102.549,
+      "duration": 0.0,
+      "text": "deal with every every letter separately"
+    },
+    {
+      "start": 1102.559,
+      "duration": 0.0,
+      "text": "deal with every every letter separately and<00:18:22.679><c> we</c><00:18:22.880><c> can</c><00:18:23.039><c> then</c><00:18:23.200><c> do</c><00:18:23.480><c> composition</c><00:18:24.280><c> where</c><00:18:24.440><c> you</c>"
+    },
+    {
+      "start": 1104.51,
+      "duration": 0.0,
+      "text": "and we can then do composition where you"
+    },
+    {
+      "start": 1104.52,
+      "duration": 0.0,
+      "text": "and we can then do composition where you know<00:18:24.720><c> that</c><00:18:24.880><c> basically</c><00:18:25.159><c> if</c><00:18:25.280><c> you</c><00:18:25.520><c> add</c><00:18:25.840><c> stuff</c>"
+    },
+    {
+      "start": 1106.11,
+      "duration": 0.0,
+      "text": "know that basically if you add stuff"
+    },
+    {
+      "start": 1106.12,
+      "duration": 0.0,
+      "text": "know that basically if you add stuff it's<00:18:26.240><c> just</c><00:18:26.360><c> the</c><00:18:26.440><c> same</c><00:18:26.640><c> thing</c><00:18:26.799><c> as</c><00:18:26.960><c> adding</c><00:18:27.440><c> every</c>"
+    },
+    {
+      "start": 1107.99,
+      "duration": 0.0,
+      "text": "it's just the same thing as adding every"
+    },
+    {
+      "start": 1108.0,
+      "duration": 0.0,
+      "text": "it's just the same thing as adding every one<00:18:28.280><c> separately</c><00:18:28.919><c> plus</c><00:18:29.200><c> like</c><00:18:29.320><c> whatever</c><00:18:29.600><c> the</c>"
+    },
+    {
+      "start": 1109.75,
+      "duration": 0.0,
+      "text": "one separately plus like whatever the"
+    },
+    {
+      "start": 1109.76,
+      "duration": 0.0,
+      "text": "one separately plus like whatever the unit<00:18:30.000><c> that</c><00:18:30.120><c> you</c><00:18:30.280><c> add</c><00:18:30.880><c> so</c><00:18:31.080><c> they</c><00:18:31.200><c> can</c><00:18:31.400><c> do</c><00:18:31.640><c> that</c><00:18:32.400><c> um</c>"
+    },
+    {
+      "start": 1112.71,
+      "duration": 0.0,
+      "text": "unit that you add so they can do that um"
+    },
+    {
+      "start": 1112.72,
+      "duration": 0.0,
+      "text": "unit that you add so they can do that um so<00:18:32.919><c> then</c><00:18:33.039><c> you</c><00:18:33.159><c> have</c><00:18:33.320><c> to</c><00:18:33.480><c> do</c><00:18:33.679><c> like</c><00:18:33.880><c> special</c>"
+    },
+    {
+      "start": 1114.35,
+      "duration": 0.0,
+      "text": "so then you have to do like special"
+    },
+    {
+      "start": 1114.36,
+      "duration": 0.0,
+      "text": "so then you have to do like special tokenization<00:18:35.360><c> and</c><00:18:35.799><c> like</c><00:18:36.159><c> one</c><00:18:36.320><c> of</c><00:18:36.440><c> the</c><00:18:36.600><c> big</c>"
+    },
+    {
+      "start": 1116.83,
+      "duration": 0.0,
+      "text": "tokenization and like one of the big"
+    },
+    {
+      "start": 1116.84,
+      "duration": 0.0,
+      "text": "tokenization and like one of the big changes<00:18:37.280><c> that</c><00:18:37.440><c> GPT</c><00:18:38.280><c> 4</c><00:18:38.880><c> did</c><00:18:39.640><c> uh</c><00:18:39.760><c> is</c><00:18:39.960><c> changing</c>"
+    },
+    {
+      "start": 1120.549,
+      "duration": 0.0,
+      "text": "changes that GPT 4 did uh is changing"
+    },
+    {
+      "start": 1120.559,
+      "duration": 0.0,
+      "text": "changes that GPT 4 did uh is changing the<00:18:40.679><c> way</c><00:18:40.840><c> that</c><00:18:40.960><c> they</c><00:18:41.159><c> tokenize</c><00:18:42.159><c> uh</c><00:18:42.320><c> code</c><00:18:42.919><c> so</c>"
+    },
+    {
+      "start": 1123.07,
+      "duration": 0.0,
+      "text": "the way that they tokenize uh code so"
+    },
+    {
+      "start": 1123.08,
+      "duration": 0.0,
+      "text": "the way that they tokenize uh code so for<00:18:43.240><c> example</c><00:18:43.840><c> uh</c><00:18:43.960><c> if</c><00:18:44.039><c> you</c><00:18:44.159><c> have</c><00:18:44.320><c> code</c><00:18:44.799><c> you</c><00:18:44.919><c> know</c>"
+    },
+    {
+      "start": 1125.029,
+      "duration": 0.0,
+      "text": "for example uh if you have code you know"
+    },
+    {
+      "start": 1125.039,
+      "duration": 0.0,
+      "text": "for example uh if you have code you know you<00:18:45.159><c> have</c><00:18:45.320><c> like</c><00:18:45.559><c> often</c><00:18:46.000><c> in</c><00:18:46.200><c> Python</c><00:18:46.520><c> these</c><00:18:46.640><c> four</c>"
+    },
+    {
+      "start": 1126.95,
+      "duration": 0.0,
+      "text": "you have like often in Python these four"
+    },
+    {
+      "start": 1126.96,
+      "duration": 0.0,
+      "text": "you have like often in Python these four spaces<00:18:47.320><c> at</c><00:18:47.400><c> the</c><00:18:47.520><c> beginning</c><00:18:48.200><c> those</c><00:18:48.360><c> were</c><00:18:48.600><c> dealt</c>"
+    },
+    {
+      "start": 1129.07,
+      "duration": 0.0,
+      "text": "spaces at the beginning those were dealt"
+    },
+    {
+      "start": 1129.08,
+      "duration": 0.0,
+      "text": "spaces at the beginning those were dealt with<00:18:49.799><c> uh</c><00:18:49.960><c> kind</c><00:18:50.120><c> of</c><00:18:50.520><c> strangely</c><00:18:51.159><c> before</c><00:18:52.080><c> um</c><00:18:52.280><c> and</c>"
+    },
+    {
+      "start": 1132.39,
+      "duration": 0.0,
+      "text": "with uh kind of strangely before um and"
+    },
+    {
+      "start": 1132.4,
+      "duration": 0.0,
+      "text": "with uh kind of strangely before um and as<00:18:52.480><c> a</c><00:18:52.600><c> result</c><00:18:53.000><c> like</c><00:18:53.120><c> the</c><00:18:53.200><c> model</c><00:18:53.559><c> couldn't</c>"
+    },
+    {
+      "start": 1133.99,
+      "duration": 0.0,
+      "text": "as a result like the model couldn't"
+    },
+    {
+      "start": 1134.0,
+      "duration": 0.0,
+      "text": "as a result like the model couldn't really<00:18:54.520><c> understand</c><00:18:55.280><c> uh</c><00:18:55.440><c> how</c><00:18:55.640><c> to</c><00:18:56.080><c> deal</c><00:18:56.320><c> with</c>"
+    },
+    {
+      "start": 1136.549,
+      "duration": 0.0,
+      "text": "really understand uh how to deal with"
+    },
+    {
+      "start": 1136.559,
+      "duration": 0.0,
+      "text": "really understand uh how to deal with code<00:18:57.320><c> uh</c><00:18:57.440><c> so</c><00:18:57.679><c> so</c><00:18:57.840><c> toiz</c><00:18:58.360><c> actually</c><00:18:58.919><c> a</c><00:18:59.039><c> lot</c><00:18:59.919><c> um</c>"
+    },
+    {
+      "start": 1140.909,
+      "duration": 0.0,
+      "text": "code uh so so toiz actually a lot um"
+    },
+    {
+      "start": 1140.919,
+      "duration": 0.0,
+      "text": "code uh so so toiz actually a lot um okay<00:19:01.520><c> so</c><00:19:01.760><c> I'll</c><00:19:01.919><c> move</c><00:19:02.120><c> on</c><00:19:02.919><c> right</c><00:19:03.039><c> now</c><00:19:03.200><c> but</c><00:19:03.320><c> we</c>"
+    },
+    {
+      "start": 1143.39,
+      "duration": 0.0,
+      "text": "okay so I'll move on right now but we"
+    },
+    {
+      "start": 1143.4,
+      "duration": 0.0,
+      "text": "okay so I'll move on right now but we can<00:19:03.520><c> come</c><00:19:03.679><c> back</c><00:19:03.840><c> later</c><00:19:04.080><c> on</c><00:19:04.200><c> token</c><00:19:04.960><c> Isis</c><00:19:05.960><c> great</c>"
+    },
+    {
+      "start": 1146.47,
+      "duration": 0.0,
+      "text": "can come back later on token Isis great"
+    },
+    {
+      "start": 1146.48,
+      "duration": 0.0,
+      "text": "can come back later on token Isis great so<00:19:06.640><c> we</c><00:19:06.799><c> talked</c><00:19:07.000><c> about</c><00:19:07.120><c> the</c><00:19:07.240><c> task</c><00:19:07.480><c> the</c><00:19:07.600><c> L</c><00:19:07.840><c> the</c>"
+    },
+    {
+      "start": 1147.95,
+      "duration": 0.0,
+      "text": "so we talked about the task the L the"
+    },
+    {
+      "start": 1147.96,
+      "duration": 0.0,
+      "text": "so we talked about the task the L the tokenizer<00:19:08.880><c> let's</c><00:19:09.080><c> talk</c><00:19:09.240><c> a</c><00:19:09.320><c> little</c><00:19:09.440><c> bit</c><00:19:09.559><c> about</c>"
+    },
+    {
+      "start": 1150.35,
+      "duration": 0.0,
+      "text": "tokenizer let's talk a little bit about"
+    },
+    {
+      "start": 1150.36,
+      "duration": 0.0,
+      "text": "tokenizer let's talk a little bit about evaluation<00:19:11.360><c> uh</c><00:19:11.480><c> so</c><00:19:11.640><c> the</c><00:19:11.760><c> way</c><00:19:11.960><c> that</c><00:19:12.080><c> LMS</c><00:19:12.480><c> are</c>"
+    },
+    {
+      "start": 1152.59,
+      "duration": 0.0,
+      "text": "evaluation uh so the way that LMS are"
+    },
+    {
+      "start": 1152.6,
+      "duration": 0.0,
+      "text": "evaluation uh so the way that LMS are usually<00:19:12.919><c> evaluated</c><00:19:13.720><c> is</c><00:19:13.880><c> what</c><00:19:14.000><c> we</c><00:19:14.159><c> call</c><00:19:14.440><c> is</c>"
+    },
+    {
+      "start": 1154.59,
+      "duration": 0.0,
+      "text": "usually evaluated is what we call is"
+    },
+    {
+      "start": 1154.6,
+      "duration": 0.0,
+      "text": "usually evaluated is what we call is using<00:19:15.080><c> what</c><00:19:15.200><c> we</c><00:19:15.320><c> call</c><00:19:15.679><c> perplexity</c><00:19:16.679><c> um</c><00:19:16.880><c> at</c><00:19:16.960><c> a</c>"
+    },
+    {
+      "start": 1157.11,
+      "duration": 0.0,
+      "text": "using what we call perplexity um at a"
+    },
+    {
+      "start": 1157.12,
+      "duration": 0.0,
+      "text": "using what we call perplexity um at a high<00:19:17.320><c> level</c><00:19:17.760><c> it's</c><00:19:17.960><c> basically</c><00:19:18.280><c> just</c><00:19:18.400><c> your</c>"
+    },
+    {
+      "start": 1158.59,
+      "duration": 0.0,
+      "text": "high level it's basically just your"
+    },
+    {
+      "start": 1158.6,
+      "duration": 0.0,
+      "text": "high level it's basically just your validation<00:19:19.080><c> loss</c><00:19:19.919><c> uh</c><00:19:20.120><c> the</c><00:19:20.320><c> slight</c><00:19:20.679><c> difference</c>"
+    },
+    {
+      "start": 1160.95,
+      "duration": 0.0,
+      "text": "validation loss uh the slight difference"
+    },
+    {
+      "start": 1160.96,
+      "duration": 0.0,
+      "text": "validation loss uh the slight difference with<00:19:21.159><c> perplexity</c><00:19:22.000><c> is</c><00:19:22.120><c> that</c><00:19:22.280><c> we</c><00:19:22.400><c> use</c><00:19:22.600><c> something</c>"
+    },
+    {
+      "start": 1162.87,
+      "duration": 0.0,
+      "text": "with perplexity is that we use something"
+    },
+    {
+      "start": 1162.88,
+      "duration": 0.0,
+      "text": "with perplexity is that we use something that<00:19:23.000><c> is</c><00:19:23.120><c> slightly</c><00:19:23.440><c> more</c><00:19:23.640><c> interpretable</c>"
+    },
+    {
+      "start": 1164.51,
+      "duration": 0.0,
+      "text": "that is slightly more interpretable"
+    },
+    {
+      "start": 1164.52,
+      "duration": 0.0,
+      "text": "that is slightly more interpretable which<00:19:24.640><c> is</c><00:19:24.799><c> that</c><00:19:24.919><c> we</c><00:19:25.039><c> use</c><00:19:25.240><c> the</c><00:19:25.520><c> average</c><00:19:26.320><c> per</c>"
+    },
+    {
+      "start": 1166.63,
+      "duration": 0.0,
+      "text": "which is that we use the average per"
+    },
+    {
+      "start": 1166.64,
+      "duration": 0.0,
+      "text": "which is that we use the average per token<00:19:27.120><c> loss</c><00:19:27.840><c> and</c><00:19:28.000><c> then</c><00:19:28.120><c> you</c><00:19:28.320><c> expon</c><00:19:28.840><c> entiate</c><00:19:29.200><c> it</c>"
+    },
+    {
+      "start": 1169.47,
+      "duration": 0.0,
+      "text": "token loss and then you expon entiate it"
+    },
+    {
+      "start": 1169.48,
+      "duration": 0.0,
+      "text": "token loss and then you expon entiate it and<00:19:29.559><c> the</c><00:19:29.679><c> reason</c><00:19:29.880><c> why</c><00:19:30.000><c> you</c><00:19:30.120><c> exponentiate</c><00:19:30.679><c> it</c>"
+    },
+    {
+      "start": 1171.07,
+      "duration": 0.0,
+      "text": "and the reason why you exponentiate it"
+    },
+    {
+      "start": 1171.08,
+      "duration": 0.0,
+      "text": "and the reason why you exponentiate it is<00:19:31.280><c> because</c><00:19:31.520><c> you</c><00:19:31.720><c> want</c><00:19:32.240><c> I</c><00:19:32.320><c> mean</c><00:19:32.480><c> the</c><00:19:32.640><c> loss</c><00:19:33.400><c> has</c>"
+    },
+    {
+      "start": 1173.51,
+      "duration": 0.0,
+      "text": "is because you want I mean the loss has"
+    },
+    {
+      "start": 1173.52,
+      "duration": 0.0,
+      "text": "is because you want I mean the loss has a<00:19:33.720><c> log</c><00:19:34.120><c> inside</c><00:19:34.600><c> and</c><00:19:34.799><c> you</c><00:19:35.440><c> like</c><00:19:35.640><c> one</c><00:19:35.880><c> humans</c><00:19:36.159><c> are</c>"
+    },
+    {
+      "start": 1176.31,
+      "duration": 0.0,
+      "text": "a log inside and you like one humans are"
+    },
+    {
+      "start": 1176.32,
+      "duration": 0.0,
+      "text": "a log inside and you like one humans are actually<00:19:36.480><c> pretty</c><00:19:36.760><c> bad</c><00:19:36.880><c> at</c><00:19:37.000><c> thinking</c><00:19:37.240><c> in</c><00:19:37.360><c> log</c>"
+    },
+    {
+      "start": 1177.63,
+      "duration": 0.0,
+      "text": "actually pretty bad at thinking in log"
+    },
+    {
+      "start": 1177.64,
+      "duration": 0.0,
+      "text": "actually pretty bad at thinking in log space<00:19:38.080><c> but</c><00:19:38.200><c> two</c><00:19:38.679><c> logs</c><00:19:39.000><c> depend</c><00:19:39.280><c> on</c><00:19:39.360><c> the</c><00:19:39.520><c> base</c><00:19:39.960><c> of</c>"
+    },
+    {
+      "start": 1180.11,
+      "duration": 0.0,
+      "text": "space but two logs depend on the base of"
+    },
+    {
+      "start": 1180.12,
+      "duration": 0.0,
+      "text": "space but two logs depend on the base of the<00:19:40.280><c> log</c><00:19:41.120><c> uh</c><00:19:41.320><c> while</c><00:19:42.080><c> when</c><00:19:42.159><c> you</c><00:19:42.320><c> exponentiate</c>"
+    },
+    {
+      "start": 1182.95,
+      "duration": 0.0,
+      "text": "the log uh while when you exponentiate"
+    },
+    {
+      "start": 1182.96,
+      "duration": 0.0,
+      "text": "the log uh while when you exponentiate you<00:19:43.080><c> basically</c><00:19:43.400><c> have</c><00:19:43.600><c> everything</c><00:19:43.919><c> in</c><00:19:44.120><c> the</c><00:19:44.919><c> uh</c>"
+    },
+    {
+      "start": 1185.11,
+      "duration": 0.0,
+      "text": "you basically have everything in the uh"
+    },
+    {
+      "start": 1185.12,
+      "duration": 0.0,
+      "text": "you basically have everything in the uh kind<00:19:45.240><c> of</c><00:19:45.400><c> the</c><00:19:45.640><c> vocabulary</c><00:19:46.280><c> size</c><00:19:46.919><c> uh</c><00:19:47.120><c> unit</c><00:19:48.120><c> um</c>"
+    },
+    {
+      "start": 1188.669,
+      "duration": 0.0,
+      "text": "kind of the vocabulary size uh unit um"
+    },
+    {
+      "start": 1188.679,
+      "duration": 0.0,
+      "text": "kind of the vocabulary size uh unit um and<00:19:48.840><c> the</c><00:19:49.039><c> average</c><00:19:49.360><c> proten</c><00:19:49.840><c> is</c><00:19:49.960><c> just</c><00:19:50.080><c> so</c><00:19:50.280><c> that</c>"
+    },
+    {
+      "start": 1190.47,
+      "duration": 0.0,
+      "text": "and the average proten is just so that"
+    },
+    {
+      "start": 1190.48,
+      "duration": 0.0,
+      "text": "and the average proten is just so that your<00:19:50.679><c> your</c><00:19:50.840><c> complexity</c><00:19:51.360><c> is</c><00:19:51.520><c> independent</c><00:19:52.240><c> of</c>"
+    },
+    {
+      "start": 1192.39,
+      "duration": 0.0,
+      "text": "your your complexity is independent of"
+    },
+    {
+      "start": 1192.4,
+      "duration": 0.0,
+      "text": "your your complexity is independent of the<00:19:52.520><c> length</c><00:19:52.840><c> of</c><00:19:52.919><c> your</c><00:19:53.080><c> sequence</c><00:19:54.000><c> um</c><00:19:54.240><c> so</c>"
+    },
+    {
+      "start": 1194.47,
+      "duration": 0.0,
+      "text": "the length of your sequence um so"
+    },
+    {
+      "start": 1194.48,
+      "duration": 0.0,
+      "text": "the length of your sequence um so perplexity<00:19:55.080><c> is</c><00:19:55.200><c> just</c><00:19:55.360><c> two</c><00:19:55.559><c> to</c><00:19:55.679><c> the</c><00:19:55.840><c> power</c><00:19:56.760><c> uh</c>"
+    },
+    {
+      "start": 1196.909,
+      "duration": 0.0,
+      "text": "perplexity is just two to the power uh"
+    },
+    {
+      "start": 1196.919,
+      "duration": 0.0,
+      "text": "perplexity is just two to the power uh average<00:19:57.280><c> of</c><00:19:57.520><c> the</c><00:19:57.640><c> loss</c><00:19:58.000><c> of</c><00:19:58.120><c> the</c><00:19:58.280><c> sequence</c>"
+    },
+    {
+      "start": 1199.909,
+      "duration": 0.0,
+      "text": "average of the loss of the sequence"
+    },
+    {
+      "start": 1199.919,
+      "duration": 0.0,
+      "text": "average of the loss of the sequence um<00:20:00.159><c> so</c><00:20:00.440><c> perplexity</c><00:20:01.440><c> is</c><00:20:01.640><c> between</c><00:20:02.080><c> one</c><00:20:02.760><c> and</c><00:20:02.960><c> the</c>"
+    },
+    {
+      "start": 1203.07,
+      "duration": 0.0,
+      "text": "um so perplexity is between one and the"
+    },
+    {
+      "start": 1203.08,
+      "duration": 0.0,
+      "text": "um so perplexity is between one and the length<00:20:03.320><c> of</c><00:20:03.480><c> the</c><00:20:03.640><c> vocabulary</c><00:20:04.360><c> of</c><00:20:04.480><c> your</c>"
+    },
+    {
+      "start": 1204.71,
+      "duration": 0.0,
+      "text": "length of the vocabulary of your"
+    },
+    {
+      "start": 1204.72,
+      "duration": 0.0,
+      "text": "length of the vocabulary of your tokenizer<00:20:05.720><c> uh</c><00:20:05.840><c> one</c><00:20:06.120><c> it's</c><00:20:06.280><c> simply</c><00:20:06.880><c> well</c><00:20:07.039><c> if</c><00:20:07.159><c> you</c>"
+    },
+    {
+      "start": 1207.43,
+      "duration": 0.0,
+      "text": "tokenizer uh one it's simply well if you"
+    },
+    {
+      "start": 1207.44,
+      "duration": 0.0,
+      "text": "tokenizer uh one it's simply well if you predict<00:20:07.880><c> perfectly</c><00:20:08.280><c> the</c><00:20:08.400><c> thing</c><00:20:08.679><c> which</c><00:20:09.240><c> uh</c>"
+    },
+    {
+      "start": 1209.789,
+      "duration": 0.0,
+      "text": "predict perfectly the thing which uh"
+    },
+    {
+      "start": 1209.799,
+      "duration": 0.0,
+      "text": "predict perfectly the thing which uh every<00:20:10.080><c> word</c><00:20:10.640><c> then</c><00:20:10.919><c> every</c><00:20:11.200><c> word</c><00:20:11.559><c> will</c><00:20:11.840><c> have</c>"
+    },
+    {
+      "start": 1212.43,
+      "duration": 0.0,
+      "text": "every word then every word will have"
+    },
+    {
+      "start": 1212.44,
+      "duration": 0.0,
+      "text": "every word then every word will have basically<00:20:12.880><c> product</c><00:20:13.320><c> of</c><00:20:13.559><c> ones</c><00:20:14.480><c> uh</c><00:20:14.600><c> so</c><00:20:14.840><c> the</c><00:20:14.960><c> best</c>"
+    },
+    {
+      "start": 1215.19,
+      "duration": 0.0,
+      "text": "basically product of ones uh so the best"
+    },
+    {
+      "start": 1215.2,
+      "duration": 0.0,
+      "text": "basically product of ones uh so the best perplexity<00:20:15.679><c> you</c><00:20:15.720><c> can</c><00:20:15.840><c> have</c><00:20:15.960><c> is</c><00:20:16.159><c> one</c><00:20:16.760><c> if</c><00:20:16.919><c> you</c>"
+    },
+    {
+      "start": 1217.11,
+      "duration": 0.0,
+      "text": "perplexity you can have is one if you"
+    },
+    {
+      "start": 1217.12,
+      "duration": 0.0,
+      "text": "perplexity you can have is one if you really<00:20:17.360><c> have</c><00:20:17.559><c> no</c><00:20:17.799><c> idea</c><00:20:18.120><c> you</c><00:20:18.320><c> basically</c>"
+    },
+    {
+      "start": 1218.71,
+      "duration": 0.0,
+      "text": "really have no idea you basically"
+    },
+    {
+      "start": 1218.72,
+      "duration": 0.0,
+      "text": "really have no idea you basically predict<00:20:19.080><c> with</c><00:20:19.280><c> one</c><00:20:19.559><c> divided</c><00:20:19.960><c> by</c><00:20:20.559><c> uh</c><00:20:20.720><c> size</c><00:20:20.960><c> of</c>"
+    },
+    {
+      "start": 1221.23,
+      "duration": 0.0,
+      "text": "predict with one divided by uh size of"
+    },
+    {
+      "start": 1221.24,
+      "duration": 0.0,
+      "text": "predict with one divided by uh size of vocabulary<00:20:22.240><c> um</c><00:20:22.480><c> and</c><00:20:22.600><c> then</c><00:20:22.720><c> you</c><00:20:22.840><c> do</c><00:20:22.960><c> simple</c>"
+    },
+    {
+      "start": 1223.27,
+      "duration": 0.0,
+      "text": "vocabulary um and then you do simple"
+    },
+    {
+      "start": 1223.28,
+      "duration": 0.0,
+      "text": "vocabulary um and then you do simple math<00:20:23.480><c> and</c><00:20:23.559><c> you</c><00:20:23.679><c> basically</c><00:20:24.000><c> get</c><00:20:24.200><c> perplexity</c><00:20:25.080><c> of</c>"
+    },
+    {
+      "start": 1225.27,
+      "duration": 0.0,
+      "text": "math and you basically get perplexity of"
+    },
+    {
+      "start": 1225.28,
+      "duration": 0.0,
+      "text": "math and you basically get perplexity of size<00:20:25.520><c> of</c><00:20:25.720><c> vocabulary</c><00:20:26.720><c> uh</c><00:20:26.799><c> so</c><00:20:26.960><c> the</c><00:20:27.080><c> intuition</c>"
+    },
+    {
+      "start": 1227.51,
+      "duration": 0.0,
+      "text": "size of vocabulary uh so the intuition"
+    },
+    {
+      "start": 1227.52,
+      "duration": 0.0,
+      "text": "size of vocabulary uh so the intuition of<00:20:27.720><c> perplexity</c><00:20:28.280><c> is</c><00:20:28.400><c> that</c><00:20:28.760><c> basically</c><00:20:29.120><c> the</c>"
+    },
+    {
+      "start": 1229.27,
+      "duration": 0.0,
+      "text": "of perplexity is that basically the"
+    },
+    {
+      "start": 1229.28,
+      "duration": 0.0,
+      "text": "of perplexity is that basically the number<00:20:29.520><c> of</c><00:20:29.720><c> tokens</c><00:20:30.120><c> that</c><00:20:30.240><c> your</c><00:20:30.400><c> model</c><00:20:30.720><c> is</c><00:20:30.880><c> kind</c>"
+    },
+    {
+      "start": 1230.99,
+      "duration": 0.0,
+      "text": "number of tokens that your model is kind"
+    },
+    {
+      "start": 1231.0,
+      "duration": 0.0,
+      "text": "number of tokens that your model is kind of<00:20:31.240><c> hesitating</c><00:20:31.799><c> between</c><00:20:32.760><c> uh</c><00:20:32.919><c> so</c><00:20:33.080><c> if</c><00:20:33.240><c> you</c><00:20:33.440><c> if</c>"
+    },
+    {
+      "start": 1233.51,
+      "duration": 0.0,
+      "text": "of hesitating between uh so if you if"
+    },
+    {
+      "start": 1233.52,
+      "duration": 0.0,
+      "text": "of hesitating between uh so if you if your<00:20:33.640><c> model</c><00:20:33.880><c> is</c><00:20:34.039><c> perfect</c><00:20:34.559><c> it</c><00:20:34.679><c> doesn't</c>"
+    },
+    {
+      "start": 1234.95,
+      "duration": 0.0,
+      "text": "your model is perfect it doesn't"
+    },
+    {
+      "start": 1234.96,
+      "duration": 0.0,
+      "text": "your model is perfect it doesn't hesitate<00:20:35.440><c> it</c><00:20:35.600><c> know</c><00:20:35.799><c> exactly</c><00:20:36.080><c> the</c><00:20:36.240><c> word</c><00:20:36.640><c> if</c><00:20:36.840><c> it</c>"
+    },
+    {
+      "start": 1237.07,
+      "duration": 0.0,
+      "text": "hesitate it know exactly the word if it"
+    },
+    {
+      "start": 1237.08,
+      "duration": 0.0,
+      "text": "hesitate it know exactly the word if it really<00:20:37.440><c> has</c><00:20:37.640><c> no</c><00:20:37.840><c> idea</c><00:20:38.360><c> then</c><00:20:38.480><c> it</c><00:20:38.679><c> hesitates</c>"
+    },
+    {
+      "start": 1239.23,
+      "duration": 0.0,
+      "text": "really has no idea then it hesitates"
+    },
+    {
+      "start": 1239.24,
+      "duration": 0.0,
+      "text": "really has no idea then it hesitates between<00:20:40.240><c> uh</c><00:20:40.760><c> all</c><00:20:40.960><c> of</c><00:20:41.120><c> the</c>"
+    },
+    {
+      "start": 1242.59,
+      "duration": 0.0,
+      "text": "between uh all of the"
+    },
+    {
+      "start": 1242.6,
+      "duration": 0.0,
+      "text": "between uh all of the vocabulary<00:20:43.600><c> uh</c><00:20:43.880><c> so</c><00:20:44.200><c> perplexity</c><00:20:45.200><c> really</c>"
+    },
+    {
+      "start": 1245.549,
+      "duration": 0.0,
+      "text": "vocabulary uh so perplexity really"
+    },
+    {
+      "start": 1245.559,
+      "duration": 0.0,
+      "text": "vocabulary uh so perplexity really improved<00:20:46.200><c> that's</c><00:20:46.520><c> perplexity</c><00:20:47.400><c> on</c><00:20:47.559><c> a</c><00:20:47.720><c> standard</c>"
+    },
+    {
+      "start": 1248.11,
+      "duration": 0.0,
+      "text": "improved that's perplexity on a standard"
+    },
+    {
+      "start": 1248.12,
+      "duration": 0.0,
+      "text": "improved that's perplexity on a standard data<00:20:48.360><c> set</c><00:20:48.520><c> between</c><00:20:48.799><c> 2017</c><00:20:49.440><c> and</c><00:20:49.840><c> 2023</c><00:20:50.840><c> it</c><00:20:51.000><c> it</c>"
+    },
+    {
+      "start": 1251.149,
+      "duration": 0.0,
+      "text": "data set between 2017 and 2023 it it"
+    },
+    {
+      "start": 1251.159,
+      "duration": 0.0,
+      "text": "data set between 2017 and 2023 it it went<00:20:51.400><c> from</c><00:20:51.760><c> kind</c><00:20:51.919><c> of</c><00:20:52.080><c> 70</c><00:20:52.720><c> tokens</c><00:20:53.360><c> to</c><00:20:53.640><c> less</c><00:20:53.799><c> than</c>"
+    },
+    {
+      "start": 1253.99,
+      "duration": 0.0,
+      "text": "went from kind of 70 tokens to less than"
+    },
+    {
+      "start": 1254.0,
+      "duration": 0.0,
+      "text": "went from kind of 70 tokens to less than 10<00:20:54.240><c> tokens</c><00:20:55.000><c> over</c><00:20:55.280><c> these</c><00:20:55.520><c> five</c><00:20:55.720><c> six</c><00:20:56.000><c> years</c><00:20:56.520><c> so</c>"
+    },
+    {
+      "start": 1256.669,
+      "duration": 0.0,
+      "text": "10 tokens over these five six years so"
+    },
+    {
+      "start": 1256.679,
+      "duration": 0.0,
+      "text": "10 tokens over these five six years so that<00:20:56.799><c> means</c><00:20:57.000><c> that</c><00:20:57.120><c> the</c><00:20:57.240><c> models</c><00:20:57.880><c> were</c>"
+    },
+    {
+      "start": 1258.19,
+      "duration": 0.0,
+      "text": "that means that the models were"
+    },
+    {
+      "start": 1258.2,
+      "duration": 0.0,
+      "text": "that means that the models were previously<00:20:58.720><c> as</c><00:20:58.880><c> dating</c><00:20:59.159><c> between</c><00:20:59.480><c> 70</c><00:21:00.120><c> words</c>"
+    },
+    {
+      "start": 1260.47,
+      "duration": 0.0,
+      "text": "previously as dating between 70 words"
+    },
+    {
+      "start": 1260.48,
+      "duration": 0.0,
+      "text": "previously as dating between 70 words every<00:21:00.720><c> time</c><00:21:01.159><c> it</c><00:21:01.280><c> was</c><00:21:01.480><c> generating</c><00:21:01.880><c> a</c><00:21:02.039><c> word</c><00:21:02.440><c> and</c>"
+    },
+    {
+      "start": 1262.549,
+      "duration": 0.0,
+      "text": "every time it was generating a word and"
+    },
+    {
+      "start": 1262.559,
+      "duration": 0.0,
+      "text": "every time it was generating a word and now<00:21:02.720><c> it's</c><00:21:03.000><c> as</c><00:21:03.159><c> dating</c><00:21:03.440><c> between</c><00:21:03.720><c> like</c><00:21:03.880><c> less</c>"
+    },
+    {
+      "start": 1264.029,
+      "duration": 0.0,
+      "text": "now it's as dating between like less"
+    },
+    {
+      "start": 1264.039,
+      "duration": 0.0,
+      "text": "now it's as dating between like less than<00:21:04.200><c> 10</c><00:21:04.440><c> words</c><00:21:05.120><c> so</c><00:21:05.240><c> that's</c><00:21:05.440><c> much</c><00:21:05.799><c> better</c>"
+    },
+    {
+      "start": 1266.789,
+      "duration": 0.0,
+      "text": "than 10 words so that's much better"
+    },
+    {
+      "start": 1266.799,
+      "duration": 0.0,
+      "text": "than 10 words so that's much better perplexity<00:21:07.320><c> is</c><00:21:07.480><c> actually</c><00:21:07.720><c> not</c><00:21:07.919><c> used</c><00:21:08.320><c> anymore</c>"
+    },
+    {
+      "start": 1268.669,
+      "duration": 0.0,
+      "text": "perplexity is actually not used anymore"
+    },
+    {
+      "start": 1268.679,
+      "duration": 0.0,
+      "text": "perplexity is actually not used anymore in<00:21:08.919><c> academic</c><00:21:09.360><c> benchmarking</c><00:21:10.200><c> mostly</c><00:21:10.559><c> because</c>"
+    },
+    {
+      "start": 1270.669,
+      "duration": 0.0,
+      "text": "in academic benchmarking mostly because"
+    },
+    {
+      "start": 1270.679,
+      "duration": 0.0,
+      "text": "in academic benchmarking mostly because it<00:21:10.799><c> depends</c><00:21:11.080><c> on</c><00:21:11.159><c> the</c><00:21:11.279><c> tokenizers</c><00:21:11.880><c> that</c><00:21:11.960><c> you</c>"
+    },
+    {
+      "start": 1272.11,
+      "duration": 0.0,
+      "text": "it depends on the tokenizers that you"
+    },
+    {
+      "start": 1272.12,
+      "duration": 0.0,
+      "text": "it depends on the tokenizers that you use<00:21:12.880><c> uh</c><00:21:12.960><c> it</c><00:21:13.120><c> depends</c><00:21:13.400><c> on</c><00:21:13.640><c> the</c><00:21:13.880><c> actual</c><00:21:14.200><c> data</c>"
+    },
+    {
+      "start": 1274.549,
+      "duration": 0.0,
+      "text": "use uh it depends on the actual data"
+    },
+    {
+      "start": 1274.559,
+      "duration": 0.0,
+      "text": "use uh it depends on the actual data that<00:21:14.679><c> people</c><00:21:14.919><c> are</c><00:21:15.080><c> evaluating</c><00:21:15.600><c> on</c><00:21:16.200><c> but</c><00:21:16.320><c> it's</c>"
+    },
+    {
+      "start": 1276.47,
+      "duration": 0.0,
+      "text": "that people are evaluating on but it's"
+    },
+    {
+      "start": 1276.48,
+      "duration": 0.0,
+      "text": "that people are evaluating on but it's still<00:21:16.720><c> very</c><00:21:16.919><c> important</c><00:21:17.240><c> for</c><00:21:17.480><c> development</c><00:21:18.240><c> of</c>"
+    },
+    {
+      "start": 1278.39,
+      "duration": 0.0,
+      "text": "still very important for development of"
+    },
+    {
+      "start": 1278.4,
+      "duration": 0.0,
+      "text": "still very important for development of llms<00:21:19.120><c> so</c><00:21:19.360><c> when</c><00:21:19.480><c> you</c><00:21:19.760><c> when</c><00:21:19.880><c> you</c><00:21:20.000><c> actually</c><00:21:20.200><c> train</c>"
+    },
+    {
+      "start": 1280.39,
+      "duration": 0.0,
+      "text": "llms so when you when you actually train"
+    },
+    {
+      "start": 1280.4,
+      "duration": 0.0,
+      "text": "llms so when you when you actually train your<00:21:20.520><c> own</c><00:21:20.720><c> llm</c><00:21:21.360><c> people</c><00:21:21.600><c> will</c><00:21:21.840><c> still</c><00:21:22.120><c> really</c>"
+    },
+    {
+      "start": 1282.31,
+      "duration": 0.0,
+      "text": "your own llm people will still really"
+    },
+    {
+      "start": 1282.32,
+      "duration": 0.0,
+      "text": "your own llm people will still really look<00:21:22.880><c> at</c><00:21:23.080><c> the</c>"
+    },
+    {
+      "start": 1284.669,
+      "duration": 0.0,
+      "text": "look at the"
+    },
+    {
+      "start": 1284.679,
+      "duration": 0.0,
+      "text": "look at the perplexity<00:21:25.679><c> uh</c><00:21:26.240><c> one</c><00:21:26.640><c> common</c><00:21:27.120><c> other</c><00:21:27.400><c> way</c><00:21:27.760><c> and</c>"
+    },
+    {
+      "start": 1288.19,
+      "duration": 0.0,
+      "text": "perplexity uh one common other way and"
+    },
+    {
+      "start": 1288.2,
+      "duration": 0.0,
+      "text": "perplexity uh one common other way and now<00:21:28.600><c> more</c><00:21:28.720><c> common</c><00:21:29.200><c> in</c><00:21:29.480><c> Academia</c><00:21:30.200><c> of</c>"
+    },
+    {
+      "start": 1290.39,
+      "duration": 0.0,
+      "text": "now more common in Academia of"
+    },
+    {
+      "start": 1290.4,
+      "duration": 0.0,
+      "text": "now more common in Academia of evaluating<00:21:30.919><c> these</c><00:21:31.039><c> llms</c><00:21:31.840><c> is</c><00:21:32.000><c> just</c><00:21:32.240><c> by</c><00:21:32.760><c> taking</c>"
+    },
+    {
+      "start": 1293.149,
+      "duration": 0.0,
+      "text": "evaluating these llms is just by taking"
+    },
+    {
+      "start": 1293.159,
+      "duration": 0.0,
+      "text": "evaluating these llms is just by taking all<00:21:33.360><c> the</c><00:21:33.559><c> classical</c><00:21:34.080><c> NLP</c><00:21:34.600><c> benchmarks</c><00:21:35.120><c> and</c>"
+    },
+    {
+      "start": 1295.23,
+      "duration": 0.0,
+      "text": "all the classical NLP benchmarks and"
+    },
+    {
+      "start": 1295.24,
+      "duration": 0.0,
+      "text": "all the classical NLP benchmarks and I'll<00:21:35.440><c> give</c><00:21:35.520><c> you</c><00:21:35.640><c> a</c><00:21:35.720><c> few</c><00:21:35.919><c> examples</c><00:21:36.279><c> later</c><00:21:37.000><c> and</c>"
+    },
+    {
+      "start": 1297.149,
+      "duration": 0.0,
+      "text": "I'll give you a few examples later and"
+    },
+    {
+      "start": 1297.159,
+      "duration": 0.0,
+      "text": "I'll give you a few examples later and just<00:21:37.320><c> kind</c><00:21:37.440><c> of</c><00:21:37.600><c> aggregating</c><00:21:38.240><c> everything</c><00:21:39.200><c> um</c>"
+    },
+    {
+      "start": 1299.43,
+      "duration": 0.0,
+      "text": "just kind of aggregating everything um"
+    },
+    {
+      "start": 1299.44,
+      "duration": 0.0,
+      "text": "just kind of aggregating everything um so<00:21:39.679><c> collect</c><00:21:40.039><c> as</c><00:21:40.159><c> many</c><00:21:40.720><c> automatically</c>"
+    },
+    {
+      "start": 1301.669,
+      "duration": 0.0,
+      "text": "so collect as many automatically"
+    },
+    {
+      "start": 1301.679,
+      "duration": 0.0,
+      "text": "so collect as many automatically evaluatable<00:21:42.520><c> benchmarks</c><00:21:43.279><c> and</c><00:21:43.480><c> just</c><00:21:43.679><c> evaluate</c>"
+    },
+    {
+      "start": 1304.19,
+      "duration": 0.0,
+      "text": "evaluatable benchmarks and just evaluate"
+    },
+    {
+      "start": 1304.2,
+      "duration": 0.0,
+      "text": "evaluatable benchmarks and just evaluate across<00:21:44.440><c> all</c><00:21:44.600><c> of</c><00:21:44.799><c> them</c><00:21:45.720><c> um</c><00:21:46.600><c> so</c><00:21:47.120><c> one</c><00:21:47.799><c> such</c><00:21:48.200><c> if</c><00:21:48.520><c> uh</c>"
+    },
+    {
+      "start": 1308.63,
+      "duration": 0.0,
+      "text": "across all of them um so one such if uh"
+    },
+    {
+      "start": 1308.64,
+      "duration": 0.0,
+      "text": "across all of them um so one such if uh or<00:21:48.919><c> actually</c><00:21:49.240><c> two</c><00:21:49.559><c> such</c><00:21:50.279><c> uh</c><00:21:50.440><c> benchmarks</c><00:21:51.279><c> of</c>"
+    },
+    {
+      "start": 1311.549,
+      "duration": 0.0,
+      "text": "or actually two such uh benchmarks of"
+    },
+    {
+      "start": 1311.559,
+      "duration": 0.0,
+      "text": "or actually two such uh benchmarks of what<00:21:51.679><c> we</c><00:21:51.799><c> call</c><00:21:52.400><c> uh</c><00:21:52.600><c> Helm</c><00:21:53.000><c> which</c><00:21:53.080><c> is</c><00:21:53.200><c> from</c>"
+    },
+    {
+      "start": 1313.35,
+      "duration": 0.0,
+      "text": "what we call uh Helm which is from"
+    },
+    {
+      "start": 1313.36,
+      "duration": 0.0,
+      "text": "what we call uh Helm which is from Stanford<00:21:54.039><c> and</c><00:21:54.200><c> another</c><00:21:54.400><c> one</c><00:21:54.520><c> is</c><00:21:54.640><c> the</c><00:21:54.760><c> hugging</c>"
+    },
+    {
+      "start": 1315.029,
+      "duration": 0.0,
+      "text": "Stanford and another one is the hugging"
+    },
+    {
+      "start": 1315.039,
+      "duration": 0.0,
+      "text": "Stanford and another one is the hugging face<00:21:55.320><c> open</c><00:21:55.720><c> LM</c><00:21:56.080><c> leader</c><00:21:56.320><c> board</c><00:21:56.600><c> which</c><00:21:56.720><c> are</c><00:21:56.840><c> the</c>"
+    },
+    {
+      "start": 1317.029,
+      "duration": 0.0,
+      "text": "face open LM leader board which are the"
+    },
+    {
+      "start": 1317.039,
+      "duration": 0.0,
+      "text": "face open LM leader board which are the probably<00:21:57.440><c> two</c><00:21:57.679><c> two</c><00:21:57.880><c> most</c><00:21:58.080><c> common</c><00:21:58.320><c> ones</c><00:21:58.799><c> right</c>"
+    },
+    {
+      "start": 1318.95,
+      "duration": 0.0,
+      "text": "probably two two most common ones right"
+    },
+    {
+      "start": 1318.96,
+      "duration": 0.0,
+      "text": "probably two two most common ones right now<00:21:59.960><c> um</c><00:22:00.400><c> so</c><00:22:00.720><c> just</c><00:22:00.840><c> to</c><00:22:01.080><c> give</c><00:22:01.159><c> you</c><00:22:01.279><c> an</c><00:22:01.440><c> idea</c><00:22:02.039><c> in</c>"
+    },
+    {
+      "start": 1322.269,
+      "duration": 0.0,
+      "text": "now um so just to give you an idea in"
+    },
+    {
+      "start": 1322.279,
+      "duration": 0.0,
+      "text": "now um so just to give you an idea in Helm<00:22:02.679><c> there</c><00:22:02.799><c> are</c><00:22:03.000><c> all</c><00:22:03.120><c> of</c><00:22:03.320><c> these</c><00:22:03.520><c> type</c><00:22:03.720><c> of</c>"
+    },
+    {
+      "start": 1323.95,
+      "duration": 0.0,
+      "text": "Helm there are all of these type of"
+    },
+    {
+      "start": 1323.96,
+      "duration": 0.0,
+      "text": "Helm there are all of these type of tasks<00:22:04.720><c> which</c><00:22:04.840><c> are</c><00:22:05.120><c> mostly</c><00:22:06.080><c> things</c><00:22:06.360><c> that</c><00:22:06.559><c> can</c>"
+    },
+    {
+      "start": 1326.669,
+      "duration": 0.0,
+      "text": "tasks which are mostly things that can"
+    },
+    {
+      "start": 1326.679,
+      "duration": 0.0,
+      "text": "tasks which are mostly things that can be<00:22:06.880><c> easily</c><00:22:07.400><c> evaluated</c><00:22:08.400><c> uh</c><00:22:08.600><c> like</c><00:22:08.880><c> question</c>"
+    },
+    {
+      "start": 1329.23,
+      "duration": 0.0,
+      "text": "be easily evaluated uh like question"
+    },
+    {
+      "start": 1329.24,
+      "duration": 0.0,
+      "text": "be easily evaluated uh like question answering<00:22:09.799><c> so</c><00:22:10.000><c> think</c><00:22:10.200><c> about</c><00:22:10.440><c> many</c><00:22:10.720><c> different</c>"
+    },
+    {
+      "start": 1331.029,
+      "duration": 0.0,
+      "text": "answering so think about many different"
+    },
+    {
+      "start": 1331.039,
+      "duration": 0.0,
+      "text": "answering so think about many different question<00:22:11.400><c> answering</c><00:22:12.279><c> uh</c><00:22:12.440><c> tasks</c><00:22:13.400><c> um</c><00:22:13.679><c> and</c><00:22:13.840><c> the</c>"
+    },
+    {
+      "start": 1334.11,
+      "duration": 0.0,
+      "text": "question answering uh tasks um and the"
+    },
+    {
+      "start": 1334.12,
+      "duration": 0.0,
+      "text": "question answering uh tasks um and the benefit<00:22:14.480><c> with</c><00:22:14.640><c> question</c><00:22:14.960><c> answering</c><00:22:15.440><c> is</c><00:22:15.559><c> that</c>"
+    },
+    {
+      "start": 1335.71,
+      "duration": 0.0,
+      "text": "benefit with question answering is that"
+    },
+    {
+      "start": 1335.72,
+      "duration": 0.0,
+      "text": "benefit with question answering is that you<00:22:15.880><c> usually</c><00:22:16.159><c> know</c><00:22:16.400><c> what</c><00:22:16.559><c> is</c><00:22:16.679><c> the</c><00:22:16.840><c> real</c><00:22:17.200><c> answer</c>"
+    },
+    {
+      "start": 1338.19,
+      "duration": 0.0,
+      "text": "you usually know what is the real answer"
+    },
+    {
+      "start": 1338.2,
+      "duration": 0.0,
+      "text": "you usually know what is the real answer um<00:22:18.600><c> so</c><00:22:18.760><c> you</c><00:22:18.880><c> can</c><00:22:19.200><c> the</c><00:22:19.320><c> way</c><00:22:19.480><c> that</c><00:22:19.600><c> you</c><00:22:19.720><c> evaluate</c>"
+    },
+    {
+      "start": 1340.07,
+      "duration": 0.0,
+      "text": "um so you can the way that you evaluate"
+    },
+    {
+      "start": 1340.08,
+      "duration": 0.0,
+      "text": "um so you can the way that you evaluate these<00:22:20.200><c> models</c><00:22:20.480><c> and</c><00:22:20.600><c> I'll</c><00:22:20.760><c> give</c><00:22:20.840><c> you</c><00:22:20.919><c> a</c>"
+    },
+    {
+      "start": 1341.029,
+      "duration": 0.0,
+      "text": "these models and I'll give you a"
+    },
+    {
+      "start": 1341.039,
+      "duration": 0.0,
+      "text": "these models and I'll give you a concrete<00:22:21.400><c> example</c><00:22:21.720><c> in</c><00:22:21.880><c> one</c><00:22:22.080><c> second</c><00:22:22.960><c> um</c><00:22:23.240><c> is</c>"
+    },
+    {
+      "start": 1343.35,
+      "duration": 0.0,
+      "text": "concrete example in one second um is"
+    },
+    {
+      "start": 1343.36,
+      "duration": 0.0,
+      "text": "concrete example in one second um is that<00:22:23.520><c> you</c><00:22:23.600><c> can</c><00:22:23.799><c> just</c><00:22:24.039><c> look</c><00:22:24.240><c> at</c><00:22:24.760><c> How</c><00:22:25.039><c> likely</c><00:22:25.760><c> the</c>"
+    },
+    {
+      "start": 1345.95,
+      "duration": 0.0,
+      "text": "that you can just look at How likely the"
+    },
+    {
+      "start": 1345.96,
+      "duration": 0.0,
+      "text": "that you can just look at How likely the language<00:22:26.320><c> model</c><00:22:26.679><c> is</c><00:22:26.840><c> to</c><00:22:27.039><c> generate</c><00:22:27.640><c> the</c><00:22:27.840><c> real</c>"
+    },
+    {
+      "start": 1348.11,
+      "duration": 0.0,
+      "text": "language model is to generate the real"
+    },
+    {
+      "start": 1348.12,
+      "duration": 0.0,
+      "text": "language model is to generate the real answer<00:22:28.799><c> compared</c><00:22:29.159><c> to</c><00:22:29.320><c> some</c><00:22:29.600><c> other</c><00:22:29.880><c> answers</c>"
+    },
+    {
+      "start": 1350.549,
+      "duration": 0.0,
+      "text": "answer compared to some other answers"
+    },
+    {
+      "start": 1350.559,
+      "duration": 0.0,
+      "text": "answer compared to some other answers and<00:22:30.720><c> that's</c><00:22:30.919><c> essentially</c><00:22:31.240><c> at</c><00:22:31.320><c> a</c><00:22:31.480><c> high</c><00:22:31.640><c> level</c>"
+    },
+    {
+      "start": 1352.07,
+      "duration": 0.0,
+      "text": "and that's essentially at a high level"
+    },
+    {
+      "start": 1352.08,
+      "duration": 0.0,
+      "text": "and that's essentially at a high level how<00:22:32.200><c> you</c><00:22:32.360><c> evaluate</c><00:22:32.760><c> these</c><00:22:32.919><c> models</c><00:22:33.840><c> um</c><00:22:34.000><c> so</c><00:22:34.159><c> to</c>"
+    },
+    {
+      "start": 1354.31,
+      "duration": 0.0,
+      "text": "how you evaluate these models um so to"
+    },
+    {
+      "start": 1354.32,
+      "duration": 0.0,
+      "text": "how you evaluate these models um so to give<00:22:34.440><c> you</c><00:22:34.520><c> a</c><00:22:34.720><c> specific</c><00:22:35.159><c> example</c><00:22:35.679><c> mlu</c><00:22:36.279><c> is</c>"
+    },
+    {
+      "start": 1356.549,
+      "duration": 0.0,
+      "text": "give you a specific example mlu is"
+    },
+    {
+      "start": 1356.559,
+      "duration": 0.0,
+      "text": "give you a specific example mlu is probably<00:22:37.159><c> the</c><00:22:37.320><c> most</c><00:22:37.600><c> common</c><00:22:38.360><c> um</c><00:22:38.960><c> academic</c>"
+    },
+    {
+      "start": 1359.51,
+      "duration": 0.0,
+      "text": "probably the most common um academic"
+    },
+    {
+      "start": 1359.52,
+      "duration": 0.0,
+      "text": "probably the most common um academic Benchmark<00:22:40.080><c> for</c>"
+    },
+    {
+      "start": 1361.07,
+      "duration": 0.0,
+      "text": "Benchmark for"
+    },
+    {
+      "start": 1361.08,
+      "duration": 0.0,
+      "text": "Benchmark for llms<00:22:42.080><c> uh</c><00:22:42.360><c> and</c><00:22:42.960><c> this</c><00:22:43.080><c> is</c><00:22:43.240><c> just</c><00:22:43.360><c> a</c><00:22:43.520><c> collection</c><00:22:44.360><c> of</c>"
+    },
+    {
+      "start": 1364.669,
+      "duration": 0.0,
+      "text": "llms uh and this is just a collection of"
+    },
+    {
+      "start": 1364.679,
+      "duration": 0.0,
+      "text": "llms uh and this is just a collection of many<00:22:45.240><c> question</c><00:22:45.559><c> and</c><00:22:45.799><c> answers</c><00:22:46.200><c> in</c><00:22:46.320><c> all</c><00:22:46.480><c> of</c>"
+    },
+    {
+      "start": 1366.669,
+      "duration": 0.0,
+      "text": "many question and answers in all of"
+    },
+    {
+      "start": 1366.679,
+      "duration": 0.0,
+      "text": "many question and answers in all of those<00:22:46.880><c> domains</c><00:22:47.520><c> for</c><00:22:47.720><c> example</c><00:22:48.120><c> College</c>"
+    },
+    {
+      "start": 1368.549,
+      "duration": 0.0,
+      "text": "those domains for example College"
+    },
+    {
+      "start": 1368.559,
+      "duration": 0.0,
+      "text": "those domains for example College medicine<00:22:49.120><c> College</c><00:22:49.600><c> physics</c><00:22:50.320><c> astronomy</c><00:22:51.240><c> and</c>"
+    },
+    {
+      "start": 1371.39,
+      "duration": 0.0,
+      "text": "medicine College physics astronomy and"
+    },
+    {
+      "start": 1371.4,
+      "duration": 0.0,
+      "text": "medicine College physics astronomy and these<00:22:51.600><c> type</c><00:22:51.799><c> of</c><00:22:51.960><c> topics</c><00:22:52.640><c> and</c><00:22:52.760><c> the</c><00:22:52.919><c> questions</c>"
+    },
+    {
+      "start": 1373.23,
+      "duration": 0.0,
+      "text": "these type of topics and the questions"
+    },
+    {
+      "start": 1373.24,
+      "duration": 0.0,
+      "text": "these type of topics and the questions are<00:22:53.440><c> things</c><00:22:53.760><c> like</c><00:22:54.120><c> so</c><00:22:54.320><c> this</c><00:22:54.480><c> in</c><00:22:54.679><c> astronomy</c>"
+    },
+    {
+      "start": 1375.39,
+      "duration": 0.0,
+      "text": "are things like so this in astronomy"
+    },
+    {
+      "start": 1375.4,
+      "duration": 0.0,
+      "text": "are things like so this in astronomy what<00:22:55.520><c> is</c><00:22:55.880><c> true</c><00:22:56.159><c> for</c><00:22:56.559><c> type</c><00:22:56.799><c> 1</c><00:22:57.080><c> a</c><00:22:57.320><c> supernova</c><00:22:58.200><c> then</c>"
+    },
+    {
+      "start": 1378.51,
+      "duration": 0.0,
+      "text": "what is true for type 1 a supernova then"
+    },
+    {
+      "start": 1378.52,
+      "duration": 0.0,
+      "text": "what is true for type 1 a supernova then you<00:22:58.720><c> give</c><00:22:59.440><c> uh</c><00:22:59.760><c> four</c><00:23:00.240><c> different</c><00:23:00.559><c> potential</c>"
+    },
+    {
+      "start": 1381.029,
+      "duration": 0.0,
+      "text": "you give uh four different potential"
+    },
+    {
+      "start": 1381.039,
+      "duration": 0.0,
+      "text": "you give uh four different potential answers<00:23:01.960><c> and</c><00:23:02.080><c> you</c><00:23:02.279><c> just</c><00:23:02.600><c> ask</c><00:23:02.880><c> the</c><00:23:03.000><c> model</c><00:23:03.600><c> which</c>"
+    },
+    {
+      "start": 1383.75,
+      "duration": 0.0,
+      "text": "answers and you just ask the model which"
+    },
+    {
+      "start": 1383.76,
+      "duration": 0.0,
+      "text": "answers and you just ask the model which one<00:23:03.919><c> is</c><00:23:04.080><c> more</c><00:23:04.320><c> likely</c><00:23:04.720><c> so</c><00:23:05.159><c> there</c><00:23:05.240><c> are</c><00:23:05.400><c> many</c>"
+    },
+    {
+      "start": 1385.63,
+      "duration": 0.0,
+      "text": "one is more likely so there are many"
+    },
+    {
+      "start": 1385.64,
+      "duration": 0.0,
+      "text": "one is more likely so there are many different<00:23:05.880><c> ways</c><00:23:06.080><c> of</c><00:23:06.200><c> doing</c><00:23:06.400><c> it</c><00:23:06.760><c> either</c><00:23:06.960><c> you</c>"
+    },
+    {
+      "start": 1387.029,
+      "duration": 0.0,
+      "text": "different ways of doing it either you"
+    },
+    {
+      "start": 1387.039,
+      "duration": 0.0,
+      "text": "different ways of doing it either you can<00:23:07.200><c> look</c><00:23:07.320><c> at</c><00:23:07.480><c> the</c><00:23:07.600><c> likelihood</c><00:23:08.360><c> of</c><00:23:08.559><c> generating</c>"
+    },
+    {
+      "start": 1389.11,
+      "duration": 0.0,
+      "text": "can look at the likelihood of generating"
+    },
+    {
+      "start": 1389.12,
+      "duration": 0.0,
+      "text": "can look at the likelihood of generating all<00:23:09.320><c> these</c><00:23:09.520><c> answers</c><00:23:10.440><c> uh</c><00:23:10.559><c> or</c><00:23:10.679><c> you</c><00:23:10.799><c> can</c><00:23:10.919><c> ask</c><00:23:11.200><c> the</c>"
+    },
+    {
+      "start": 1391.31,
+      "duration": 0.0,
+      "text": "all these answers uh or you can ask the"
+    },
+    {
+      "start": 1391.32,
+      "duration": 0.0,
+      "text": "all these answers uh or you can ask the model<00:23:11.640><c> which</c><00:23:11.799><c> one</c><00:23:11.960><c> is</c><00:23:12.080><c> the</c><00:23:12.200><c> most</c><00:23:12.440><c> likely</c><00:23:13.200><c> uh</c><00:23:13.279><c> so</c>"
+    },
+    {
+      "start": 1393.39,
+      "duration": 0.0,
+      "text": "model which one is the most likely uh so"
+    },
+    {
+      "start": 1393.4,
+      "duration": 0.0,
+      "text": "model which one is the most likely uh so there<00:23:13.480><c> are</c><00:23:13.640><c> different</c><00:23:13.840><c> ways</c><00:23:14.039><c> that</c><00:23:14.120><c> you</c><00:23:14.200><c> can</c>"
+    },
+    {
+      "start": 1394.31,
+      "duration": 0.0,
+      "text": "there are different ways that you can"
+    },
+    {
+      "start": 1394.32,
+      "duration": 0.0,
+      "text": "there are different ways that you can promp<00:23:14.600><c> the</c><00:23:14.720><c> model</c><00:23:15.039><c> but</c><00:23:15.240><c> at</c><00:23:15.320><c> a</c><00:23:15.480><c> high</c><00:23:15.679><c> level</c><00:23:16.279><c> you</c>"
+    },
+    {
+      "start": 1396.43,
+      "duration": 0.0,
+      "text": "promp the model but at a high level you"
+    },
+    {
+      "start": 1396.44,
+      "duration": 0.0,
+      "text": "promp the model but at a high level you know<00:23:16.640><c> which</c><00:23:16.799><c> one</c><00:23:16.919><c> is</c><00:23:17.039><c> correct</c><00:23:17.559><c> and</c><00:23:17.679><c> there</c><00:23:17.760><c> are</c>"
+    },
+    {
+      "start": 1397.909,
+      "duration": 0.0,
+      "text": "know which one is correct and there are"
+    },
+    {
+      "start": 1397.919,
+      "duration": 0.0,
+      "text": "know which one is correct and there are three<00:23:18.120><c> other</c><00:23:18.400><c> mistakes</c><00:23:19.400><c> um</c><00:23:20.320><c> yes</c><00:23:21.320><c> kind</c>"
+    },
+    {
+      "start": 1402.11,
+      "duration": 0.0,
+      "text": "three other mistakes um yes kind"
+    },
+    {
+      "start": 1402.12,
+      "duration": 0.0,
+      "text": "three other mistakes um yes kind creating<00:23:22.520><c> is</c><00:23:22.679><c> like</c><00:23:22.919><c> unconstrained</c><00:23:23.720><c> text</c><00:23:24.120><c> as</c>"
+    },
+    {
+      "start": 1404.269,
+      "duration": 0.0,
+      "text": "creating is like unconstrained text as"
+    },
+    {
+      "start": 1404.279,
+      "duration": 0.0,
+      "text": "creating is like unconstrained text as the<00:23:24.440><c> output</c><00:23:25.039><c> yeah</c><00:23:25.640><c> how</c><00:23:25.760><c> do</c><00:23:25.880><c> you</c><00:23:26.360><c> evaluate</c><00:23:26.760><c> a</c>"
+    },
+    {
+      "start": 1406.87,
+      "duration": 0.0,
+      "text": "the output yeah how do you evaluate a"
+    },
+    {
+      "start": 1406.88,
+      "duration": 0.0,
+      "text": "the output yeah how do you evaluate a model<00:23:27.320><c> if</c><00:23:27.799><c> it</c><00:23:27.960><c> give</c><00:23:28.120><c> something</c><00:23:28.559><c> that's</c><00:23:29.039><c> you</c>"
+    },
+    {
+      "start": 1409.19,
+      "duration": 0.0,
+      "text": "model if it give something that's you"
+    },
+    {
+      "start": 1409.2,
+      "duration": 0.0,
+      "text": "model if it give something that's you know<00:23:29.919><c> semantically</c><00:23:30.919><c> completely</c><00:23:31.559><c> identical</c>"
+    },
+    {
+      "start": 1412.549,
+      "duration": 0.0,
+      "text": "know semantically completely identical"
+    },
+    {
+      "start": 1412.559,
+      "duration": 0.0,
+      "text": "know semantically completely identical but<00:23:33.080><c> is</c><00:23:33.279><c> not</c><00:23:33.559><c> the</c><00:23:33.760><c> exact</c><00:23:34.120><c> token</c><00:23:34.520><c> list</c><00:23:34.799><c> that</c>"
+    },
+    {
+      "start": 1415.11,
+      "duration": 0.0,
+      "text": "but is not the exact token list that"
+    },
+    {
+      "start": 1415.12,
+      "duration": 0.0,
+      "text": "but is not the exact token list that expect<00:23:35.760><c> yeah</c><00:23:35.960><c> so</c><00:23:36.120><c> that's</c><00:23:36.240><c> a</c><00:23:36.520><c> great</c><00:23:36.760><c> question</c>"
+    },
+    {
+      "start": 1417.23,
+      "duration": 0.0,
+      "text": "expect yeah so that's a great question"
+    },
+    {
+      "start": 1417.24,
+      "duration": 0.0,
+      "text": "expect yeah so that's a great question I'll<00:23:37.480><c> talk</c><00:23:37.679><c> more</c><00:23:37.880><c> about</c><00:23:38.159><c> that</c><00:23:38.360><c> later</c><00:23:39.000><c> here</c><00:23:39.159><c> in</c>"
+    },
+    {
+      "start": 1419.31,
+      "duration": 0.0,
+      "text": "I'll talk more about that later here in"
+    },
+    {
+      "start": 1419.32,
+      "duration": 0.0,
+      "text": "I'll talk more about that later here in this<00:23:39.520><c> case</c><00:23:39.760><c> we</c><00:23:39.919><c> don't</c><00:23:40.120><c> do</c><00:23:40.400><c> unconstrained</c><00:23:41.400><c> so</c>"
+    },
+    {
+      "start": 1421.549,
+      "duration": 0.0,
+      "text": "this case we don't do unconstrained so"
+    },
+    {
+      "start": 1421.559,
+      "duration": 0.0,
+      "text": "this case we don't do unconstrained so the<00:23:41.679><c> way</c><00:23:41.799><c> you</c><00:23:41.919><c> would</c><00:23:42.120><c> evaluate</c><00:23:42.640><c> MML</c><00:23:43.640><c> is</c>"
+    },
+    {
+      "start": 1423.83,
+      "duration": 0.0,
+      "text": "the way you would evaluate MML is"
+    },
+    {
+      "start": 1423.84,
+      "duration": 0.0,
+      "text": "the way you would evaluate MML is basically<00:23:44.279><c> either</c><00:23:44.600><c> you</c><00:23:45.320><c> you</c><00:23:45.600><c> ask</c><00:23:45.840><c> the</c><00:23:46.000><c> first</c>"
+    },
+    {
+      "start": 1426.269,
+      "duration": 0.0,
+      "text": "basically either you you ask the first"
+    },
+    {
+      "start": 1426.279,
+      "duration": 0.0,
+      "text": "basically either you you ask the first question<00:23:46.919><c> and</c><00:23:47.039><c> then</c><00:23:47.159><c> you</c><00:23:47.320><c> look</c><00:23:47.440><c> at</c><00:23:47.600><c> the</c>"
+    },
+    {
+      "start": 1427.95,
+      "duration": 0.0,
+      "text": "question and then you look at the"
+    },
+    {
+      "start": 1427.96,
+      "duration": 0.0,
+      "text": "question and then you look at the likelihood<00:23:48.960><c> of</c><00:23:49.120><c> the</c><00:23:49.279><c> model</c><00:23:49.600><c> generating</c><00:23:50.159><c> a</c><00:23:50.720><c> the</c>"
+    },
+    {
+      "start": 1430.87,
+      "duration": 0.0,
+      "text": "likelihood of the model generating a the"
+    },
+    {
+      "start": 1430.88,
+      "duration": 0.0,
+      "text": "likelihood of the model generating a the likelihood<00:23:51.320><c> of</c><00:23:51.400><c> the</c><00:23:51.520><c> model</c><00:23:51.760><c> generating</c><00:23:52.240><c> b</c><00:23:52.720><c> c</c>"
+    },
+    {
+      "start": 1433.029,
+      "duration": 0.0,
+      "text": "likelihood of the model generating b c"
+    },
+    {
+      "start": 1433.039,
+      "duration": 0.0,
+      "text": "likelihood of the model generating b c and<00:23:53.240><c> d</c><00:23:53.600><c> and</c><00:23:53.720><c> you</c><00:23:53.840><c> look</c><00:23:53.960><c> at</c><00:23:54.159><c> which</c><00:23:54.279><c> one</c><00:23:54.400><c> is</c><00:23:54.520><c> the</c>"
+    },
+    {
+      "start": 1434.669,
+      "duration": 0.0,
+      "text": "and d and you look at which one is the"
+    },
+    {
+      "start": 1434.679,
+      "duration": 0.0,
+      "text": "and d and you look at which one is the most<00:23:54.880><c> likely</c><00:23:55.520><c> or</c><00:23:55.799><c> you</c><00:23:55.880><c> can</c><00:23:56.080><c> as</c><00:23:56.279><c> the</c><00:23:56.440><c> model</c><00:23:57.039><c> out</c>"
+    },
+    {
+      "start": 1437.19,
+      "duration": 0.0,
+      "text": "most likely or you can as the model out"
+    },
+    {
+      "start": 1437.2,
+      "duration": 0.0,
+      "text": "most likely or you can as the model out of<00:23:57.400><c> ABC</c><00:23:57.960><c> d</c><00:23:58.520><c> which</c><00:23:58.640><c> one</c><00:23:58.799><c> is</c><00:23:58.919><c> the</c><00:23:59.080><c> most</c><00:23:59.320><c> likely</c>"
+    },
+    {
+      "start": 1439.75,
+      "duration": 0.0,
+      "text": "of ABC d which one is the most likely"
+    },
+    {
+      "start": 1439.76,
+      "duration": 0.0,
+      "text": "of ABC d which one is the most likely and<00:23:59.840><c> you</c><00:24:00.000><c> look</c><00:24:00.159><c> at</c><00:24:00.440><c> whe</c><00:24:00.960><c> the</c><00:24:01.080><c> to</c><00:24:01.279><c> the</c><00:24:01.400><c> most</c>"
+    },
+    {
+      "start": 1441.59,
+      "duration": 0.0,
+      "text": "and you look at whe the to the most"
+    },
+    {
+      "start": 1441.6,
+      "duration": 0.0,
+      "text": "and you look at whe the to the most likely<00:24:01.840><c> next</c><00:24:02.080><c> token</c><00:24:02.360><c> is</c><00:24:02.520><c> A</c><00:24:02.679><c> B</c><00:24:02.919><c> C</c><00:24:03.159><c> or</c><00:24:03.320><c> D</c><00:24:04.000><c> so</c><00:24:04.400><c> uh</c>"
+    },
+    {
+      "start": 1444.47,
+      "duration": 0.0,
+      "text": "likely next token is A B C or D so uh"
+    },
+    {
+      "start": 1444.48,
+      "duration": 0.0,
+      "text": "likely next token is A B C or D so uh you<00:24:04.559><c> can</c><00:24:04.760><c> strain</c><00:24:05.159><c> the</c><00:24:05.279><c> model</c><00:24:05.760><c> to</c><00:24:05.919><c> say</c><00:24:06.080><c> it</c><00:24:06.200><c> can</c>"
+    },
+    {
+      "start": 1446.43,
+      "duration": 0.0,
+      "text": "you can strain the model to say it can"
+    },
+    {
+      "start": 1446.44,
+      "duration": 0.0,
+      "text": "you can strain the model to say it can only<00:24:06.679><c> answer</c><00:24:07.039><c> these</c><00:24:07.200><c> four</c><00:24:07.919><c> things</c><00:24:08.919><c> you</c><00:24:09.039><c> say</c>"
+    },
+    {
+      "start": 1449.19,
+      "duration": 0.0,
+      "text": "only answer these four things you say"
+    },
+    {
+      "start": 1449.2,
+      "duration": 0.0,
+      "text": "only answer these four things you say you<00:24:09.360><c> constraint</c><00:24:09.880><c> the</c><00:24:10.039><c> model</c><00:24:10.679><c> you</c><00:24:10.880><c> mean</c><00:24:11.440><c> you</c>"
+    },
+    {
+      "start": 1451.59,
+      "duration": 0.0,
+      "text": "you constraint the model you mean you"
+    },
+    {
+      "start": 1451.6,
+      "duration": 0.0,
+      "text": "you constraint the model you mean you constraint<00:24:12.200><c> The</c><00:24:12.320><c> Prompt</c><00:24:12.600><c> or</c><00:24:12.720><c> do</c><00:24:12.799><c> you</c><00:24:12.919><c> mean</c><00:24:13.360><c> of</c>"
+    },
+    {
+      "start": 1453.59,
+      "duration": 0.0,
+      "text": "constraint The Prompt or do you mean of"
+    },
+    {
+      "start": 1453.6,
+      "duration": 0.0,
+      "text": "constraint The Prompt or do you mean of its<00:24:13.799><c> whole</c><00:24:14.080><c> probability</c><00:24:14.640><c> distribution</c>"
+    },
+    {
+      "start": 1455.43,
+      "duration": 0.0,
+      "text": "its whole probability distribution"
+    },
+    {
+      "start": 1455.44,
+      "duration": 0.0,
+      "text": "its whole probability distribution outputs<00:24:16.080><c> you</c><00:24:16.400><c> only</c><00:24:16.799><c> comparing</c><00:24:17.440><c> the</c><00:24:17.600><c> outputs</c>"
+    },
+    {
+      "start": 1458.51,
+      "duration": 0.0,
+      "text": "outputs you only comparing the outputs"
+    },
+    {
+      "start": 1458.52,
+      "duration": 0.0,
+      "text": "outputs you only comparing the outputs like<00:24:18.600><c> you're</c><00:24:18.760><c> only</c><00:24:18.960><c> comparing</c><00:24:19.360><c> the</c>"
+    },
+    {
+      "start": 1460.029,
+      "duration": 0.0,
+      "text": "like you're only comparing the"
+    },
+    {
+      "start": 1460.039,
+      "duration": 0.0,
+      "text": "like you're only comparing the a<00:24:21.039><c> so</c><00:24:21.440><c> uh</c><00:24:21.559><c> in</c><00:24:21.679><c> the</c><00:24:21.840><c> second</c><00:24:22.120><c> case</c><00:24:22.400><c> I</c><00:24:22.559><c> gave</c><00:24:22.720><c> you</c>"
+    },
+    {
+      "start": 1462.99,
+      "duration": 0.0,
+      "text": "a so uh in the second case I gave you"
+    },
+    {
+      "start": 1463.0,
+      "duration": 0.0,
+      "text": "a so uh in the second case I gave you you<00:24:23.080><c> would</c><00:24:23.240><c> do</c><00:24:23.440><c> exactly</c><00:24:23.880><c> the</c><00:24:24.200><c> I</c><00:24:24.400><c> actually</c><00:24:24.600><c> you</c>"
+    },
+    {
+      "start": 1464.669,
+      "duration": 0.0,
+      "text": "you would do exactly the I actually you"
+    },
+    {
+      "start": 1464.679,
+      "duration": 0.0,
+      "text": "you would do exactly the I actually you would<00:24:24.840><c> do</c><00:24:25.039><c> both</c><00:24:25.279><c> you</c><00:24:25.360><c> would</c><00:24:25.520><c> prompt</c><00:24:25.799><c> the</c><00:24:25.880><c> model</c>"
+    },
+    {
+      "start": 1466.23,
+      "duration": 0.0,
+      "text": "would do both you would prompt the model"
+    },
+    {
+      "start": 1466.24,
+      "duration": 0.0,
+      "text": "would do both you would prompt the model saying<00:24:26.480><c> ABC</c><00:24:26.880><c> or</c><00:24:27.039><c> D</c><00:24:27.399><c> plus</c><00:24:27.679><c> you</c><00:24:27.799><c> would</c><00:24:27.919><c> constrain</c>"
+    },
+    {
+      "start": 1468.59,
+      "duration": 0.0,
+      "text": "saying ABC or D plus you would constrain"
+    },
+    {
+      "start": 1468.6,
+      "duration": 0.0,
+      "text": "saying ABC or D plus you would constrain to<00:24:28.840><c> only</c><00:24:29.480><c> uh</c><00:24:29.679><c> look</c><00:24:29.919><c> at</c><00:24:30.200><c> these</c><00:24:30.399><c> two</c><00:24:30.720><c> these</c><00:24:30.919><c> four</c>"
+    },
+    {
+      "start": 1471.19,
+      "duration": 0.0,
+      "text": "to only uh look at these two these four"
+    },
+    {
+      "start": 1471.2,
+      "duration": 0.0,
+      "text": "to only uh look at these two these four tokens<00:24:32.120><c> in</c><00:24:32.240><c> the</c><00:24:32.440><c> first</c><00:24:32.720><c> case</c><00:24:32.880><c> you</c><00:24:33.000><c> don't</c><00:24:33.200><c> even</c>"
+    },
+    {
+      "start": 1473.35,
+      "duration": 0.0,
+      "text": "tokens in the first case you don't even"
+    },
+    {
+      "start": 1473.36,
+      "duration": 0.0,
+      "text": "tokens in the first case you don't even need<00:24:33.480><c> to</c><00:24:33.600><c> generate</c><00:24:34.039><c> anything</c><00:24:34.600><c> so</c><00:24:34.760><c> in</c><00:24:34.840><c> the</c>"
+    },
+    {
+      "start": 1474.95,
+      "duration": 0.0,
+      "text": "need to generate anything so in the"
+    },
+    {
+      "start": 1474.96,
+      "duration": 0.0,
+      "text": "need to generate anything so in the first<00:24:35.159><c> case</c><00:24:35.320><c> you</c><00:24:35.520><c> literally</c><00:24:35.919><c> just</c><00:24:36.080><c> look</c><00:24:36.559><c> given</c>"
+    },
+    {
+      "start": 1476.789,
+      "duration": 0.0,
+      "text": "first case you literally just look given"
+    },
+    {
+      "start": 1476.799,
+      "duration": 0.0,
+      "text": "first case you literally just look given that<00:24:36.919><c> it's</c><00:24:37.000><c> a</c><00:24:37.120><c> language</c><00:24:37.440><c> model</c><00:24:37.960><c> it</c><00:24:38.080><c> can</c><00:24:38.200><c> give</c><00:24:38.320><c> a</c>"
+    },
+    {
+      "start": 1478.669,
+      "duration": 0.0,
+      "text": "that it's a language model it can give a"
+    },
+    {
+      "start": 1478.679,
+      "duration": 0.0,
+      "text": "that it's a language model it can give a distribution<00:24:39.200><c> over</c><00:24:39.440><c> sentences</c><00:24:40.120><c> you</c><00:24:40.279><c> just</c>"
+    },
+    {
+      "start": 1480.47,
+      "duration": 0.0,
+      "text": "distribution over sentences you just"
+    },
+    {
+      "start": 1480.48,
+      "duration": 0.0,
+      "text": "distribution over sentences you just look<00:24:40.640><c> at</c><00:24:41.039><c> what</c><00:24:41.159><c> is</c><00:24:41.880><c> the</c><00:24:42.080><c> likelihood</c><00:24:42.520><c> of</c>"
+    },
+    {
+      "start": 1482.63,
+      "duration": 0.0,
+      "text": "look at what is the likelihood of"
+    },
+    {
+      "start": 1482.64,
+      "duration": 0.0,
+      "text": "look at what is the likelihood of generating<00:24:43.559><c> all</c><00:24:43.760><c> of</c><00:24:43.960><c> these</c><00:24:44.159><c> words</c><00:24:45.120><c> what</c><00:24:45.240><c> is</c>"
+    },
+    {
+      "start": 1485.35,
+      "duration": 0.0,
+      "text": "generating all of these words what is"
+    },
+    {
+      "start": 1485.36,
+      "duration": 0.0,
+      "text": "generating all of these words what is the<00:24:45.520><c> likelihood</c><00:24:45.960><c> of</c><00:24:46.080><c> generating</c><00:24:46.799><c> the</c><00:24:47.039><c> second</c>"
+    },
+    {
+      "start": 1487.43,
+      "duration": 0.0,
+      "text": "the likelihood of generating the second"
+    },
+    {
+      "start": 1487.44,
+      "duration": 0.0,
+      "text": "the likelihood of generating the second choice<00:24:48.320><c> and</c><00:24:48.399><c> you</c><00:24:48.600><c> just</c><00:24:48.760><c> look</c><00:24:48.960><c> at</c><00:24:49.159><c> whether</c><00:24:49.360><c> the</c>"
+    },
+    {
+      "start": 1489.549,
+      "duration": 0.0,
+      "text": "choice and you just look at whether the"
+    },
+    {
+      "start": 1489.559,
+      "duration": 0.0,
+      "text": "choice and you just look at whether the most<00:24:49.840><c> likely</c><00:24:50.960><c> sentence</c><00:24:51.960><c> is</c><00:24:52.200><c> actually</c><00:24:52.559><c> the</c>"
+    },
+    {
+      "start": 1492.71,
+      "duration": 0.0,
+      "text": "most likely sentence is actually the"
+    },
+    {
+      "start": 1492.72,
+      "duration": 0.0,
+      "text": "most likely sentence is actually the real<00:24:53.440><c> answer</c><00:24:54.440><c> so</c><00:24:54.600><c> you</c><00:24:54.679><c> don't</c><00:24:54.960><c> actually</c><00:24:55.440><c> sample</c>"
+    },
+    {
+      "start": 1495.87,
+      "duration": 0.0,
+      "text": "real answer so you don't actually sample"
+    },
+    {
+      "start": 1495.88,
+      "duration": 0.0,
+      "text": "real answer so you don't actually sample from<00:24:56.120><c> it</c><00:24:56.279><c> you</c><00:24:56.480><c> really</c><00:24:56.720><c> just</c><00:24:56.960><c> use</c><00:24:57.559><c> P</c><00:24:57.799><c> of</c><00:24:58.000><c> x</c><00:24:58.399><c> one</c>"
+    },
+    {
+      "start": 1498.51,
+      "duration": 0.0,
+      "text": "from it you really just use P of x one"
+    },
+    {
+      "start": 1498.52,
+      "duration": 0.0,
+      "text": "from it you really just use P of x one to<00:24:58.760><c> excel</c><00:24:59.679><c> does</c><00:24:59.799><c> that</c><00:24:59.960><c> make</c><00:25:00.360><c> sense</c><00:25:01.360><c> uh</c><00:25:01.600><c> that</c>"
+    },
+    {
+      "start": 1501.75,
+      "duration": 0.0,
+      "text": "to excel does that make sense uh that"
+    },
+    {
+      "start": 1501.76,
+      "duration": 0.0,
+      "text": "to excel does that make sense uh that being<00:25:02.000><c> said</c><00:25:02.440><c> evaluation</c><00:25:03.080><c> of</c><00:25:03.399><c> open-ended</c>"
+    },
+    {
+      "start": 1504.35,
+      "duration": 0.0,
+      "text": "being said evaluation of open-ended"
+    },
+    {
+      "start": 1504.36,
+      "duration": 0.0,
+      "text": "being said evaluation of open-ended questions<00:25:05.080><c> is</c><00:25:05.240><c> something</c><00:25:05.440><c> we're</c><00:25:05.559><c> going</c><00:25:05.640><c> to</c>"
+    },
+    {
+      "start": 1505.789,
+      "duration": 0.0,
+      "text": "questions is something we're going to"
+    },
+    {
+      "start": 1505.799,
+      "duration": 0.0,
+      "text": "questions is something we're going to talk<00:25:06.000><c> about</c><00:25:06.200><c> later</c><00:25:06.880><c> and</c><00:25:07.000><c> is</c><00:25:07.200><c> actually</c><00:25:07.440><c> really</c>"
+    },
+    {
+      "start": 1507.63,
+      "duration": 0.0,
+      "text": "talk about later and is actually really"
+    },
+    {
+      "start": 1507.64,
+      "duration": 0.0,
+      "text": "talk about later and is actually really important<00:25:08.120><c> and</c><00:25:08.279><c> really</c><00:25:08.960><c> challenging</c><00:25:09.960><c> yes</c>"
+    },
+    {
+      "start": 1510.789,
+      "duration": 0.0,
+      "text": "important and really challenging yes"
+    },
+    {
+      "start": 1510.799,
+      "duration": 0.0,
+      "text": "important and really challenging yes earlier<00:25:11.120><c> you</c><00:25:11.279><c> mentioned</c><00:25:11.640><c> that</c><00:25:12.000><c> um</c><00:25:12.240><c> like</c><00:25:13.080><c> um</c>"
+    },
+    {
+      "start": 1513.31,
+      "duration": 0.0,
+      "text": "earlier you mentioned that um like um"
+    },
+    {
+      "start": 1513.32,
+      "duration": 0.0,
+      "text": "earlier you mentioned that um like um metrics<00:25:13.679><c> like</c><00:25:14.000><c> flexity</c><00:25:14.480><c> are</c><00:25:14.640><c> not</c><00:25:15.520><c> are</c><00:25:15.679><c> not</c>"
+    },
+    {
+      "start": 1515.87,
+      "duration": 0.0,
+      "text": "metrics like flexity are not are not"
+    },
+    {
+      "start": 1515.88,
+      "duration": 0.0,
+      "text": "metrics like flexity are not are not like<00:25:16.240><c> usually</c><00:25:16.600><c> used</c><00:25:16.919><c> because</c><00:25:17.120><c> it</c><00:25:17.279><c> depends</c><00:25:17.600><c> on</c>"
+    },
+    {
+      "start": 1517.83,
+      "duration": 0.0,
+      "text": "like usually used because it depends on"
+    },
+    {
+      "start": 1517.84,
+      "duration": 0.0,
+      "text": "like usually used because it depends on like<00:25:18.200><c> how</c><00:25:18.320><c> you</c><00:25:18.440><c> do</c><00:25:18.559><c> your</c><00:25:18.720><c> terization</c><00:25:19.520><c> some</c>"
+    },
+    {
+      "start": 1519.83,
+      "duration": 0.0,
+      "text": "like how you do your terization some"
+    },
+    {
+      "start": 1519.84,
+      "duration": 0.0,
+      "text": "like how you do your terization some design<00:25:20.159><c> choices</c><00:25:20.919><c> I</c><00:25:21.000><c> was</c><00:25:21.120><c> wondering</c><00:25:21.480><c> if</c><00:25:21.559><c> you</c>"
+    },
+    {
+      "start": 1521.669,
+      "duration": 0.0,
+      "text": "design choices I was wondering if you"
+    },
+    {
+      "start": 1521.679,
+      "duration": 0.0,
+      "text": "design choices I was wondering if you could<00:25:21.840><c> speak</c><00:25:22.080><c> more</c><00:25:22.240><c> to</c><00:25:22.640><c> that</c><00:25:23.640><c> oh</c><00:25:24.240><c> um</c><00:25:24.760><c> yeah</c><00:25:25.080><c> so</c>"
+    },
+    {
+      "start": 1525.549,
+      "duration": 0.0,
+      "text": "could speak more to that oh um yeah so"
+    },
+    {
+      "start": 1525.559,
+      "duration": 0.0,
+      "text": "could speak more to that oh um yeah so think<00:25:25.799><c> about</c><00:25:26.039><c> perplexity</c><00:25:26.679><c> I</c><00:25:26.760><c> told</c><00:25:26.919><c> you</c>"
+    },
+    {
+      "start": 1527.07,
+      "duration": 0.0,
+      "text": "think about perplexity I told you"
+    },
+    {
+      "start": 1527.08,
+      "duration": 0.0,
+      "text": "think about perplexity I told you perplexity<00:25:27.600><c> is</c><00:25:27.720><c> between</c><00:25:28.000><c> one</c><00:25:28.600><c> and</c><00:25:28.840><c> vocabulary</c>"
+    },
+    {
+      "start": 1529.47,
+      "duration": 0.0,
+      "text": "perplexity is between one and vocabulary"
+    },
+    {
+      "start": 1529.48,
+      "duration": 0.0,
+      "text": "perplexity is between one and vocabulary size<00:25:30.159><c> so</c><00:25:30.360><c> now</c><00:25:30.559><c> imagine</c><00:25:31.039><c> that</c><00:25:31.279><c> Chad</c><00:25:31.559><c> GPT</c><00:25:32.120><c> uses</c><00:25:32.399><c> a</c>"
+    },
+    {
+      "start": 1532.549,
+      "duration": 0.0,
+      "text": "size so now imagine that Chad GPT uses a"
+    },
+    {
+      "start": 1532.559,
+      "duration": 0.0,
+      "text": "size so now imagine that Chad GPT uses a tokenizer<00:25:33.559><c> that</c><00:25:33.760><c> has</c><00:25:33.960><c> like</c><00:25:34.120><c> 10,000</c><00:25:34.640><c> tokens</c>"
+    },
+    {
+      "start": 1535.31,
+      "duration": 0.0,
+      "text": "tokenizer that has like 10,000 tokens"
+    },
+    {
+      "start": 1535.32,
+      "duration": 0.0,
+      "text": "tokenizer that has like 10,000 tokens but<00:25:35.799><c> Gemini</c><00:25:36.480><c> from</c><00:25:36.679><c> Google</c><00:25:36.960><c> uses</c><00:25:37.240><c> a</c><00:25:37.399><c> tokenizer</c>"
+    },
+    {
+      "start": 1537.909,
+      "duration": 0.0,
+      "text": "but Gemini from Google uses a tokenizer"
+    },
+    {
+      "start": 1537.919,
+      "duration": 0.0,
+      "text": "but Gemini from Google uses a tokenizer that<00:25:38.080><c> had</c><00:25:38.799><c> 100,000</c><00:25:39.799><c> uh</c><00:25:40.399><c> potential</c><00:25:40.840><c> tokens</c>"
+    },
+    {
+      "start": 1541.71,
+      "duration": 0.0,
+      "text": "that had 100,000 uh potential tokens"
+    },
+    {
+      "start": 1541.72,
+      "duration": 0.0,
+      "text": "that had 100,000 uh potential tokens then<00:25:41.960><c> actually</c><00:25:42.200><c> the</c><00:25:42.360><c> Gemini</c><00:25:42.840><c> one</c><00:25:43.399><c> will</c><00:25:44.039><c> will</c>"
+    },
+    {
+      "start": 1544.51,
+      "duration": 0.0,
+      "text": "then actually the Gemini one will will"
+    },
+    {
+      "start": 1544.52,
+      "duration": 0.0,
+      "text": "then actually the Gemini one will will have<00:25:44.919><c> like</c><00:25:45.039><c> the</c><00:25:45.240><c> upper</c><00:25:45.480><c> bound</c><00:25:45.760><c> of</c><00:25:46.000><c> the</c><00:25:46.240><c> the</c>"
+    },
+    {
+      "start": 1546.389,
+      "duration": 0.0,
+      "text": "have like the upper bound of the the"
+    },
+    {
+      "start": 1546.399,
+      "duration": 0.0,
+      "text": "have like the upper bound of the the perplexity<00:25:46.880><c> that</c><00:25:46.960><c> you</c><00:25:47.039><c> can</c><00:25:47.200><c> get</c><00:25:47.360><c> is</c><00:25:47.520><c> actually</c>"
+    },
+    {
+      "start": 1547.789,
+      "duration": 0.0,
+      "text": "perplexity that you can get is actually"
+    },
+    {
+      "start": 1547.799,
+      "duration": 0.0,
+      "text": "perplexity that you can get is actually worse<00:25:48.039><c> for</c><00:25:48.240><c> Gemini</c><00:25:49.200><c> than</c><00:25:49.679><c> for</c><00:25:49.919><c> Chad</c><00:25:50.200><c> GPT</c><00:25:50.840><c> does</c>"
+    },
+    {
+      "start": 1550.99,
+      "duration": 0.0,
+      "text": "worse for Gemini than for Chad GPT does"
+    },
+    {
+      "start": 1551.0,
+      "duration": 0.0,
+      "text": "worse for Gemini than for Chad GPT does that<00:25:51.159><c> make</c><00:25:51.320><c> sense</c><00:25:52.320><c> so</c><00:25:52.600><c> that's</c><00:25:52.799><c> just</c><00:25:52.919><c> an</c><00:25:53.159><c> idea</c>"
+    },
+    {
+      "start": 1554.149,
+      "duration": 0.0,
+      "text": "that make sense so that's just an idea"
+    },
+    {
+      "start": 1554.159,
+      "duration": 0.0,
+      "text": "that make sense so that's just an idea it's<00:25:54.320><c> actually</c><00:25:54.480><c> a</c><00:25:54.559><c> little</c><00:25:54.679><c> bit</c><00:25:54.840><c> more</c>"
+    },
+    {
+      "start": 1554.99,
+      "duration": 0.0,
+      "text": "it's actually a little bit more"
+    },
+    {
+      "start": 1555.0,
+      "duration": 0.0,
+      "text": "it's actually a little bit more complicated<00:25:55.440><c> than</c><00:25:55.559><c> that</c><00:25:55.679><c> but</c><00:25:55.799><c> that's</c><00:25:55.960><c> just</c>"
+    },
+    {
+      "start": 1556.11,
+      "duration": 0.0,
+      "text": "complicated than that but that's just"
+    },
+    {
+      "start": 1556.12,
+      "duration": 0.0,
+      "text": "complicated than that but that's just like<00:25:56.320><c> one</c><00:25:57.080><c> uh</c><00:25:57.200><c> first</c><00:25:57.480><c> or</c><00:25:57.720><c> the</c><00:25:57.840><c> bit</c><00:25:58.000><c> of</c><00:25:58.399><c> you</c><00:25:58.480><c> can</c>"
+    },
+    {
+      "start": 1558.59,
+      "duration": 0.0,
+      "text": "like one uh first or the bit of you can"
+    },
+    {
+      "start": 1558.6,
+      "duration": 0.0,
+      "text": "like one uh first or the bit of you can see<00:25:58.760><c> that</c><00:25:59.240><c> the</c><00:25:59.360><c> tokenizer</c><00:26:00.039><c> actually</c>"
+    },
+    {
+      "start": 1561.23,
+      "duration": 0.0,
+      "text": "see that the tokenizer actually"
+    },
+    {
+      "start": 1561.24,
+      "duration": 0.0,
+      "text": "see that the tokenizer actually matters<00:26:02.240><c> um</c>"
+    },
+    {
+      "start": 1564.75,
+      "duration": 0.0,
+      "text": "matters um"
+    },
+    {
+      "start": 1564.76,
+      "duration": 0.0,
+      "text": "matters um great<00:26:05.760><c> okay</c><00:26:06.279><c> so</c><00:26:06.559><c> evaluation</c><00:26:07.080><c> challenges</c>"
+    },
+    {
+      "start": 1567.83,
+      "duration": 0.0,
+      "text": "great okay so evaluation challenges"
+    },
+    {
+      "start": 1567.84,
+      "duration": 0.0,
+      "text": "great okay so evaluation challenges there<00:26:07.960><c> are</c><00:26:08.120><c> many</c><00:26:08.480><c> I'll</c><00:26:08.679><c> just</c><00:26:08.840><c> talk</c><00:26:09.039><c> about</c><00:26:09.279><c> two</c>"
+    },
+    {
+      "start": 1569.549,
+      "duration": 0.0,
+      "text": "there are many I'll just talk about two"
+    },
+    {
+      "start": 1569.559,
+      "duration": 0.0,
+      "text": "there are many I'll just talk about two really<00:26:09.799><c> briefly</c><00:26:10.760><c> uh</c><00:26:10.919><c> one</c><00:26:11.240><c> as</c><00:26:11.360><c> I</c><00:26:11.480><c> told</c><00:26:11.679><c> you</c>"
+    },
+    {
+      "start": 1571.95,
+      "duration": 0.0,
+      "text": "really briefly uh one as I told you"
+    },
+    {
+      "start": 1571.96,
+      "duration": 0.0,
+      "text": "really briefly uh one as I told you there<00:26:12.080><c> are</c><00:26:12.200><c> two</c><00:26:12.399><c> ways</c><00:26:12.600><c> of</c><00:26:12.720><c> doing</c><00:26:13.000><c> evaluation</c>"
+    },
+    {
+      "start": 1573.549,
+      "duration": 0.0,
+      "text": "there are two ways of doing evaluation"
+    },
+    {
+      "start": 1573.559,
+      "duration": 0.0,
+      "text": "there are two ways of doing evaluation for<00:26:13.919><c> these</c><00:26:14.120><c> mlu</c><00:26:14.919><c> actually</c><00:26:15.080><c> there</c><00:26:15.200><c> are</c><00:26:15.320><c> many</c>"
+    },
+    {
+      "start": 1575.51,
+      "duration": 0.0,
+      "text": "for these mlu actually there are many"
+    },
+    {
+      "start": 1575.52,
+      "duration": 0.0,
+      "text": "for these mlu actually there are many more<00:26:15.679><c> than</c><00:26:15.799><c> two</c><00:26:16.000><c> but</c><00:26:16.120><c> I</c><00:26:16.240><c> give</c><00:26:16.360><c> you</c><00:26:16.480><c> two</c>"
+    },
+    {
+      "start": 1576.789,
+      "duration": 0.0,
+      "text": "more than two but I give you two"
+    },
+    {
+      "start": 1576.799,
+      "duration": 0.0,
+      "text": "more than two but I give you two examples<00:26:17.799><c> um</c><00:26:18.240><c> and</c><00:26:18.840><c> it</c><00:26:19.000><c> happens</c><00:26:19.399><c> that</c><00:26:19.679><c> for</c><00:26:19.840><c> a</c>"
+    },
+    {
+      "start": 1579.95,
+      "duration": 0.0,
+      "text": "examples um and it happens that for a"
+    },
+    {
+      "start": 1579.96,
+      "duration": 0.0,
+      "text": "examples um and it happens that for a long<00:26:20.240><c> time</c><00:26:20.440><c> even</c><00:26:20.640><c> though</c><00:26:20.799><c> that</c><00:26:20.919><c> was</c><00:26:21.039><c> a</c><00:26:21.200><c> very</c>"
+    },
+    {
+      "start": 1581.43,
+      "duration": 0.0,
+      "text": "long time even though that was a very"
+    },
+    {
+      "start": 1581.44,
+      "duration": 0.0,
+      "text": "long time even though that was a very classical<00:26:21.840><c> Benchmark</c><00:26:22.320><c> that</c><00:26:22.480><c> everyone</c><00:26:22.720><c> used</c>"
+    },
+    {
+      "start": 1583.63,
+      "duration": 0.0,
+      "text": "classical Benchmark that everyone used"
+    },
+    {
+      "start": 1583.64,
+      "duration": 0.0,
+      "text": "classical Benchmark that everyone used uh<00:26:23.799><c> actually</c><00:26:24.640><c> different</c><00:26:25.640><c> uh</c><00:26:26.200><c> different</c>"
+    },
+    {
+      "start": 1586.63,
+      "duration": 0.0,
+      "text": "uh actually different uh different"
+    },
+    {
+      "start": 1586.64,
+      "duration": 0.0,
+      "text": "uh actually different uh different companies<00:26:27.120><c> and</c><00:26:27.360><c> different</c><00:26:27.919><c> um</c><00:26:28.600><c> different</c><00:26:29.279><c> uh</c>"
+    },
+    {
+      "start": 1589.99,
+      "duration": 0.0,
+      "text": "companies and different um different uh"
+    },
+    {
+      "start": 1590.0,
+      "duration": 0.0,
+      "text": "companies and different um different uh uh<00:26:30.559><c> different</c><00:26:30.840><c> organization</c><00:26:31.679><c> were</c><00:26:31.919><c> actually</c>"
+    },
+    {
+      "start": 1592.149,
+      "duration": 0.0,
+      "text": "uh different organization were actually"
+    },
+    {
+      "start": 1592.159,
+      "duration": 0.0,
+      "text": "uh different organization were actually using<00:26:32.480><c> different</c><00:26:32.799><c> ways</c><00:26:33.320><c> of</c><00:26:33.520><c> evaluating</c><00:26:34.159><c> mlu</c>"
+    },
+    {
+      "start": 1595.07,
+      "duration": 0.0,
+      "text": "using different ways of evaluating mlu"
+    },
+    {
+      "start": 1595.08,
+      "duration": 0.0,
+      "text": "using different ways of evaluating mlu and<00:26:35.240><c> as</c><00:26:35.320><c> a</c><00:26:35.520><c> result</c><00:26:35.840><c> you</c><00:26:36.000><c> could</c><00:26:36.200><c> you</c><00:26:36.399><c> get</c>"
+    },
+    {
+      "start": 1596.789,
+      "duration": 0.0,
+      "text": "and as a result you could you get"
+    },
+    {
+      "start": 1596.799,
+      "duration": 0.0,
+      "text": "and as a result you could you get completely<00:26:37.279><c> different</c><00:26:37.559><c> results</c><00:26:37.880><c> for</c><00:26:38.000><c> example</c>"
+    },
+    {
+      "start": 1598.31,
+      "duration": 0.0,
+      "text": "completely different results for example"
+    },
+    {
+      "start": 1598.32,
+      "duration": 0.0,
+      "text": "completely different results for example Lama"
+    },
+    {
+      "start": 1599.83,
+      "duration": 0.0,
+      "text": "Lama"
+    },
+    {
+      "start": 1599.84,
+      "duration": 0.0,
+      "text": "Lama 65b<00:26:40.840><c> uh</c><00:26:40.960><c> which</c><00:26:41.120><c> was</c><00:26:41.320><c> the</c><00:26:41.480><c> first</c><00:26:41.720><c> model</c><00:26:42.240><c> of</c><00:26:42.480><c> meta</c>"
+    },
+    {
+      "start": 1602.87,
+      "duration": 0.0,
+      "text": "65b uh which was the first model of meta"
+    },
+    {
+      "start": 1602.88,
+      "duration": 0.0,
+      "text": "65b uh which was the first model of meta in<00:26:42.960><c> the</c><00:26:43.080><c> Lama</c><00:26:43.480><c> series</c><00:26:44.320><c> uh</c><00:26:44.559><c> had</c><00:26:45.080><c> on</c><00:26:45.320><c> Helm</c><00:26:46.120><c> 63.7</c>"
+    },
+    {
+      "start": 1607.11,
+      "duration": 0.0,
+      "text": "in the Lama series uh had on Helm 63.7"
+    },
+    {
+      "start": 1607.12,
+      "duration": 0.0,
+      "text": "in the Lama series uh had on Helm 63.7 accuracy<00:26:47.960><c> but</c><00:26:48.120><c> on</c><00:26:48.399><c> this</c><00:26:48.679><c> other</c><00:26:49.600><c> um</c><00:26:50.159><c> Benchmark</c>"
+    },
+    {
+      "start": 1610.909,
+      "duration": 0.0,
+      "text": "accuracy but on this other um Benchmark"
+    },
+    {
+      "start": 1610.919,
+      "duration": 0.0,
+      "text": "accuracy but on this other um Benchmark had<00:26:51.080><c> like</c>"
+    },
+    {
+      "start": 1611.95,
+      "duration": 0.0,
+      "text": "had like"
+    },
+    {
+      "start": 1611.96,
+      "duration": 0.0,
+      "text": "had like 48.8<00:26:52.960><c> um</c><00:26:53.320><c> so</c><00:26:53.559><c> really</c><00:26:53.760><c> the</c><00:26:53.919><c> way</c><00:26:54.520><c> that</c><00:26:54.640><c> you</c>"
+    },
+    {
+      "start": 1614.75,
+      "duration": 0.0,
+      "text": "48.8 um so really the way that you"
+    },
+    {
+      "start": 1614.76,
+      "duration": 0.0,
+      "text": "48.8 um so really the way that you evaluate<00:26:55.240><c> and</c><00:26:55.320><c> this</c><00:26:55.440><c> is</c><00:26:55.559><c> not</c><00:26:55.720><c> even</c><00:26:56.080><c> talking</c>"
+    },
+    {
+      "start": 1616.35,
+      "duration": 0.0,
+      "text": "evaluate and this is not even talking"
+    },
+    {
+      "start": 1616.36,
+      "duration": 0.0,
+      "text": "evaluate and this is not even talking about<00:26:56.600><c> prompting</c><00:26:57.120><c> this</c><00:26:57.240><c> is</c><00:26:57.480><c> really</c><00:26:57.720><c> just</c><00:26:57.919><c> kind</c>"
+    },
+    {
+      "start": 1618.029,
+      "duration": 0.0,
+      "text": "about prompting this is really just kind"
+    },
+    {
+      "start": 1618.039,
+      "duration": 0.0,
+      "text": "about prompting this is really just kind of<00:26:58.399><c> the</c><00:26:58.559><c> the</c><00:26:58.679><c> way</c><00:26:58.880><c> that</c><00:26:59.000><c> you</c><00:26:59.200><c> evaluate</c><00:26:59.960><c> the</c><00:27:00.640><c> uh</c>"
+    },
+    {
+      "start": 1620.75,
+      "duration": 0.0,
+      "text": "of the the way that you evaluate the uh"
+    },
+    {
+      "start": 1620.76,
+      "duration": 0.0,
+      "text": "of the the way that you evaluate the uh the<00:27:00.880><c> models</c><00:27:01.240><c> prompting</c><00:27:01.640><c> is</c><00:27:01.799><c> another</c><00:27:02.120><c> issue</c><00:27:02.600><c> so</c>"
+    },
+    {
+      "start": 1622.789,
+      "duration": 0.0,
+      "text": "the models prompting is another issue so"
+    },
+    {
+      "start": 1622.799,
+      "duration": 0.0,
+      "text": "the models prompting is another issue so really<00:27:03.039><c> there</c><00:27:03.120><c> are</c><00:27:03.159><c> a</c><00:27:03.279><c> lot</c><00:27:03.399><c> of</c>"
+    },
+    {
+      "start": 1623.71,
+      "duration": 0.0,
+      "text": "really there are a lot of"
+    },
+    {
+      "start": 1623.72,
+      "duration": 0.0,
+      "text": "really there are a lot of inconsistencies<00:27:04.720><c> it's</c><00:27:04.919><c> not</c><00:27:05.159><c> as</c><00:27:05.440><c> easy</c><00:27:06.080><c> as</c><00:27:06.200><c> it</c>"
+    },
+    {
+      "start": 1626.35,
+      "duration": 0.0,
+      "text": "inconsistencies it's not as easy as it"
+    },
+    {
+      "start": 1626.36,
+      "duration": 0.0,
+      "text": "inconsistencies it's not as easy as it looks<00:27:07.240><c> uh</c><00:27:07.399><c> first</c><00:27:07.679><c> thing</c><00:27:08.159><c> yeah</c><00:27:08.320><c> sorry</c><00:27:08.960><c> how</c><00:27:09.120><c> can</c>"
+    },
+    {
+      "start": 1629.23,
+      "duration": 0.0,
+      "text": "looks uh first thing yeah sorry how can"
+    },
+    {
+      "start": 1629.24,
+      "duration": 0.0,
+      "text": "looks uh first thing yeah sorry how can we<00:27:09.399><c> make</c><00:27:09.520><c> sure</c><00:27:09.760><c> that</c><00:27:09.919><c> all</c><00:27:10.080><c> these</c><00:27:10.240><c> models</c><00:27:10.559><c> AR</c>"
+    },
+    {
+      "start": 1630.83,
+      "duration": 0.0,
+      "text": "we make sure that all these models AR"
+    },
+    {
+      "start": 1630.84,
+      "duration": 0.0,
+      "text": "we make sure that all these models AR trained<00:27:11.159><c> on</c><00:27:11.320><c> The</c><00:27:11.480><c> Benchmark</c><00:27:12.440><c> okay</c><00:27:13.440><c> second</c>"
+    },
+    {
+      "start": 1633.75,
+      "duration": 0.0,
+      "text": "trained on The Benchmark okay second"
+    },
+    {
+      "start": 1633.76,
+      "duration": 0.0,
+      "text": "trained on The Benchmark okay second thing<00:27:14.080><c> this</c><00:27:14.200><c> is</c><00:27:14.320><c> a</c><00:27:14.440><c> great</c><00:27:14.679><c> question</c><00:27:15.399><c> uh</c><00:27:15.520><c> chain</c>"
+    },
+    {
+      "start": 1635.83,
+      "duration": 0.0,
+      "text": "thing this is a great question uh chain"
+    },
+    {
+      "start": 1635.84,
+      "duration": 0.0,
+      "text": "thing this is a great question uh chain test<00:27:16.440><c> contamination</c><00:27:17.440><c> uh</c><00:27:17.640><c> this</c><00:27:17.760><c> is</c><00:27:18.000><c> something</c>"
+    },
+    {
+      "start": 1638.549,
+      "duration": 0.0,
+      "text": "test contamination uh this is something"
+    },
+    {
+      "start": 1638.559,
+      "duration": 0.0,
+      "text": "test contamination uh this is something which<00:27:19.039><c> I</c><00:27:19.120><c> would</c><00:27:19.360><c> say</c><00:27:19.760><c> is</c><00:27:20.240><c> really</c><00:27:20.679><c> important</c><00:27:21.279><c> in</c>"
+    },
+    {
+      "start": 1642.19,
+      "duration": 0.0,
+      "text": "which I would say is really important in"
+    },
+    {
+      "start": 1642.2,
+      "duration": 0.0,
+      "text": "which I would say is really important in Academia<00:27:23.200><c> in</c><00:27:23.960><c> uh</c><00:27:24.240><c> given</c><00:27:24.440><c> that</c><00:27:24.600><c> the</c><00:27:24.760><c> talk</c><00:27:24.919><c> is</c>"
+    },
+    {
+      "start": 1645.11,
+      "duration": 0.0,
+      "text": "Academia in uh given that the talk is"
+    },
+    {
+      "start": 1645.12,
+      "duration": 0.0,
+      "text": "Academia in uh given that the talk is mostly<00:27:25.480><c> about</c><00:27:25.679><c> training</c><00:27:26.000><c> large</c><00:27:26.320><c> language</c>"
+    },
+    {
+      "start": 1646.63,
+      "duration": 0.0,
+      "text": "mostly about training large language"
+    },
+    {
+      "start": 1646.64,
+      "duration": 0.0,
+      "text": "mostly about training large language models<00:27:27.640><c> uh</c><00:27:27.720><c> for</c><00:27:28.159><c> companies</c><00:27:28.440><c> it's</c><00:27:28.600><c> maybe</c><00:27:28.840><c> not</c>"
+    },
+    {
+      "start": 1649.07,
+      "duration": 0.0,
+      "text": "models uh for companies it's maybe not"
+    },
+    {
+      "start": 1649.08,
+      "duration": 0.0,
+      "text": "models uh for companies it's maybe not that<00:27:29.240><c> important</c><00:27:29.720><c> CU</c><00:27:29.960><c> they</c><00:27:30.159><c> know</c><00:27:30.919><c> what</c><00:27:31.120><c> they</c>"
+    },
+    {
+      "start": 1651.31,
+      "duration": 0.0,
+      "text": "that important CU they know what they"
+    },
+    {
+      "start": 1651.32,
+      "duration": 0.0,
+      "text": "that important CU they know what they trained<00:27:31.720><c> on</c><00:27:32.720><c> uh</c><00:27:33.320><c> for</c><00:27:33.760><c> us</c><00:27:34.279><c> we</c><00:27:34.440><c> have</c><00:27:34.600><c> no</c><00:27:34.760><c> idea</c><00:27:35.320><c> so</c>"
+    },
+    {
+      "start": 1655.47,
+      "duration": 0.0,
+      "text": "trained on uh for us we have no idea so"
+    },
+    {
+      "start": 1655.48,
+      "duration": 0.0,
+      "text": "trained on uh for us we have no idea so for<00:27:35.640><c> us</c><00:27:35.799><c> it's</c><00:27:35.919><c> a</c><00:27:36.080><c> real</c><00:27:36.440><c> problem</c><00:27:37.240><c> uh</c><00:27:37.360><c> so</c><00:27:37.559><c> there</c>"
+    },
+    {
+      "start": 1657.63,
+      "duration": 0.0,
+      "text": "for us it's a real problem uh so there"
+    },
+    {
+      "start": 1657.64,
+      "duration": 0.0,
+      "text": "for us it's a real problem uh so there are<00:27:37.840><c> many</c><00:27:38.120><c> different</c><00:27:38.480><c> ways</c><00:27:38.880><c> of</c><00:27:39.039><c> trying</c><00:27:39.320><c> to</c>"
+    },
+    {
+      "start": 1659.549,
+      "duration": 0.0,
+      "text": "are many different ways of trying to"
+    },
+    {
+      "start": 1659.559,
+      "duration": 0.0,
+      "text": "are many different ways of trying to test<00:27:40.279><c> whether</c><00:27:41.240><c> uh</c><00:27:41.519><c> the</c><00:27:41.799><c> test</c><00:27:42.120><c> set</c><00:27:42.840><c> sorry</c>"
+    },
+    {
+      "start": 1663.11,
+      "duration": 0.0,
+      "text": "test whether uh the test set sorry"
+    },
+    {
+      "start": 1663.12,
+      "duration": 0.0,
+      "text": "test whether uh the test set sorry whether<00:27:43.320><c> the</c><00:27:43.480><c> test</c><00:27:43.679><c> set</c><00:27:43.880><c> was</c><00:27:44.039><c> actually</c><00:27:44.320><c> in</c><00:27:44.440><c> the</c>"
+    },
+    {
+      "start": 1664.509,
+      "duration": 0.0,
+      "text": "whether the test set was actually in the"
+    },
+    {
+      "start": 1664.519,
+      "duration": 0.0,
+      "text": "whether the test set was actually in the training<00:27:44.840><c> Set</c><00:27:45.640><c> uh</c><00:27:45.960><c> one</c><00:27:46.559><c> kind</c><00:27:46.679><c> of</c><00:27:47.399><c> cute</c><00:27:47.760><c> trick</c>"
+    },
+    {
+      "start": 1668.59,
+      "duration": 0.0,
+      "text": "training Set uh one kind of cute trick"
+    },
+    {
+      "start": 1668.6,
+      "duration": 0.0,
+      "text": "training Set uh one kind of cute trick um<00:27:49.159><c> that</c><00:27:49.399><c> people</c><00:27:50.240><c> uh</c><00:27:50.880><c> in</c><00:27:51.080><c> in</c><00:27:51.159><c> the</c><00:27:51.320><c> lab</c><00:27:51.720><c> on</c><00:27:51.960><c> T</c><00:27:52.399><c> lab</c>"
+    },
+    {
+      "start": 1672.59,
+      "duration": 0.0,
+      "text": "um that people uh in in the lab on T lab"
+    },
+    {
+      "start": 1672.6,
+      "duration": 0.0,
+      "text": "um that people uh in in the lab on T lab have<00:27:52.720><c> found</c><00:27:53.120><c> is</c><00:27:53.240><c> that</c><00:27:53.399><c> what</c><00:27:53.480><c> you</c><00:27:53.559><c> can</c><00:27:53.720><c> do</c><00:27:54.320><c> is</c>"
+    },
+    {
+      "start": 1674.47,
+      "duration": 0.0,
+      "text": "have found is that what you can do is"
+    },
+    {
+      "start": 1674.48,
+      "duration": 0.0,
+      "text": "have found is that what you can do is that<00:27:54.799><c> given</c><00:27:55.039><c> that</c><00:27:55.279><c> most</c><00:27:55.480><c> of</c><00:27:55.640><c> the</c><00:27:55.880><c> data</c><00:27:56.200><c> set</c>"
+    },
+    {
+      "start": 1676.35,
+      "duration": 0.0,
+      "text": "that given that most of the data set"
+    },
+    {
+      "start": 1676.36,
+      "duration": 0.0,
+      "text": "that given that most of the data set online<00:27:56.919><c> are</c><00:27:57.120><c> not</c><00:27:57.360><c> randomized</c>"
+    },
+    {
+      "start": 1678.63,
+      "duration": 0.0,
+      "text": "online are not randomized"
+    },
+    {
+      "start": 1678.64,
+      "duration": 0.0,
+      "text": "online are not randomized you<00:27:58.760><c> can</c><00:27:59.000><c> just</c><00:27:59.240><c> look</c><00:27:59.720><c> at</c><00:28:00.399><c> and</c><00:28:00.519><c> in</c><00:28:00.640><c> that</c>"
+    },
+    {
+      "start": 1680.789,
+      "duration": 0.0,
+      "text": "you can just look at and in that"
+    },
+    {
+      "start": 1680.799,
+      "duration": 0.0,
+      "text": "you can just look at and in that language<00:28:01.080><c> models</c><00:28:01.440><c> what</c><00:28:01.519><c> they</c><00:28:01.640><c> do</c><00:28:01.760><c> is</c><00:28:01.880><c> just</c>"
+    },
+    {
+      "start": 1682.029,
+      "duration": 0.0,
+      "text": "language models what they do is just"
+    },
+    {
+      "start": 1682.039,
+      "duration": 0.0,
+      "text": "language models what they do is just predict<00:28:02.480><c> the</c><00:28:02.640><c> next</c><00:28:02.919><c> word</c><00:28:03.720><c> um</c><00:28:03.840><c> you</c><00:28:03.960><c> can</c><00:28:04.120><c> just</c>"
+    },
+    {
+      "start": 1684.269,
+      "duration": 0.0,
+      "text": "predict the next word um you can just"
+    },
+    {
+      "start": 1684.279,
+      "duration": 0.0,
+      "text": "predict the next word um you can just look<00:28:04.480><c> at</c><00:28:04.880><c> the</c><00:28:05.039><c> entire</c><00:28:05.519><c> test</c><00:28:05.799><c> Set</c><00:28:06.720><c> uh</c><00:28:06.919><c> what</c><00:28:07.080><c> if</c>"
+    },
+    {
+      "start": 1687.23,
+      "duration": 0.0,
+      "text": "look at the entire test Set uh what if"
+    },
+    {
+      "start": 1687.24,
+      "duration": 0.0,
+      "text": "look at the entire test Set uh what if you<00:28:07.440><c> generate</c><00:28:08.240><c> all</c><00:28:08.480><c> the</c><00:28:08.640><c> examples</c><00:28:09.559><c> in</c><00:28:09.799><c> order</c>"
+    },
+    {
+      "start": 1690.669,
+      "duration": 0.0,
+      "text": "you generate all the examples in order"
+    },
+    {
+      "start": 1690.679,
+      "duration": 0.0,
+      "text": "you generate all the examples in order versus<00:28:11.600><c> all</c><00:28:11.840><c> the</c><00:28:12.080><c> examples</c><00:28:12.640><c> in</c><00:28:12.760><c> a</c><00:28:12.919><c> different</c>"
+    },
+    {
+      "start": 1693.19,
+      "duration": 0.0,
+      "text": "versus all the examples in a different"
+    },
+    {
+      "start": 1693.2,
+      "duration": 0.0,
+      "text": "versus all the examples in a different order<00:28:13.960><c> and</c><00:28:14.320><c> if</c><00:28:14.440><c> it's</c><00:28:14.679><c> more</c><00:28:14.880><c> likely</c><00:28:15.159><c> to</c>"
+    },
+    {
+      "start": 1695.31,
+      "duration": 0.0,
+      "text": "order and if it's more likely to"
+    },
+    {
+      "start": 1695.32,
+      "duration": 0.0,
+      "text": "order and if it's more likely to generate<00:28:15.679><c> a</c><00:28:15.840><c> thing</c><00:28:16.240><c> in</c><00:28:16.440><c> order</c><00:28:17.080><c> given</c><00:28:17.320><c> that</c>"
+    },
+    {
+      "start": 1697.43,
+      "duration": 0.0,
+      "text": "generate a thing in order given that"
+    },
+    {
+      "start": 1697.44,
+      "duration": 0.0,
+      "text": "generate a thing in order given that there's<00:28:17.640><c> no</c><00:28:17.880><c> real</c><00:28:18.360><c> order</c><00:28:18.880><c> there</c><00:28:19.360><c> then</c><00:28:19.480><c> it</c>"
+    },
+    {
+      "start": 1699.59,
+      "duration": 0.0,
+      "text": "there's no real order there then it"
+    },
+    {
+      "start": 1699.6,
+      "duration": 0.0,
+      "text": "there's no real order there then it means<00:28:19.840><c> that</c><00:28:20.000><c> probably</c><00:28:20.279><c> was</c><00:28:20.399><c> in</c><00:28:20.480><c> a</c><00:28:20.600><c> training</c>"
+    },
+    {
+      "start": 1700.909,
+      "duration": 0.0,
+      "text": "means that probably was in a training"
+    },
+    {
+      "start": 1700.919,
+      "duration": 0.0,
+      "text": "means that probably was in a training set<00:28:21.440><c> does</c><00:28:21.559><c> that</c><00:28:21.679><c> make</c><00:28:21.919><c> sense</c><00:28:22.919><c> um</c><00:28:23.159><c> so</c><00:28:23.440><c> there</c><00:28:23.519><c> are</c>"
+    },
+    {
+      "start": 1703.669,
+      "duration": 0.0,
+      "text": "set does that make sense um so there are"
+    },
+    {
+      "start": 1703.679,
+      "duration": 0.0,
+      "text": "set does that make sense um so there are many<00:28:23.880><c> that's</c><00:28:24.039><c> like</c><00:28:24.200><c> one</c><00:28:24.320><c> of</c><00:28:24.480><c> them</c><00:28:24.760><c> there</c><00:28:24.840><c> are</c>"
+    },
+    {
+      "start": 1704.95,
+      "duration": 0.0,
+      "text": "many that's like one of them there are"
+    },
+    {
+      "start": 1704.96,
+      "duration": 0.0,
+      "text": "many that's like one of them there are many<00:28:25.159><c> other</c><00:28:25.320><c> ways</c><00:28:25.519><c> of</c><00:28:25.640><c> doing</c><00:28:25.880><c> it</c><00:28:26.159><c> train</c><00:28:26.480><c> test</c>"
+    },
+    {
+      "start": 1707.11,
+      "duration": 0.0,
+      "text": "many other ways of doing it train test"
+    },
+    {
+      "start": 1707.12,
+      "duration": 0.0,
+      "text": "many other ways of doing it train test contamination<00:28:27.880><c> again</c><00:28:28.320><c> not</c><00:28:28.519><c> that</c><00:28:28.640><c> important</c>"
+    },
+    {
+      "start": 1708.909,
+      "duration": 0.0,
+      "text": "contamination again not that important"
+    },
+    {
+      "start": 1708.919,
+      "duration": 0.0,
+      "text": "contamination again not that important for<00:28:29.120><c> development</c><00:28:29.720><c> really</c><00:28:29.960><c> important</c><00:28:30.279><c> for</c>"
+    },
+    {
+      "start": 1710.47,
+      "duration": 0.0,
+      "text": "for development really important for"
+    },
+    {
+      "start": 1710.48,
+      "duration": 0.0,
+      "text": "for development really important for academic"
+    },
+    {
+      "start": 1712.269,
+      "duration": 0.0,
+      "text": "academic"
+    },
+    {
+      "start": 1712.279,
+      "duration": 0.0,
+      "text": "academic benchmarking<00:28:33.279><c> great</c><00:28:33.679><c> so</c><00:28:33.840><c> there</c><00:28:33.919><c> are</c><00:28:34.039><c> many</c>"
+    },
+    {
+      "start": 1714.23,
+      "duration": 0.0,
+      "text": "benchmarking great so there are many"
+    },
+    {
+      "start": 1714.24,
+      "duration": 0.0,
+      "text": "benchmarking great so there are many other<00:28:34.399><c> challenges</c><00:28:34.919><c> but</c><00:28:35.200><c> uh</c><00:28:35.679><c> I'll</c><00:28:35.880><c> move</c><00:28:36.080><c> on</c><00:28:36.320><c> for</c>"
+    },
+    {
+      "start": 1716.71,
+      "duration": 0.0,
+      "text": "other challenges but uh I'll move on for"
+    },
+    {
+      "start": 1716.72,
+      "duration": 0.0,
+      "text": "other challenges but uh I'll move on for now<00:28:37.720><c> great</c><00:28:38.640><c> data</c><00:28:39.640><c> um</c><00:28:40.519><c> so</c><00:28:40.760><c> data</c><00:28:41.039><c> is</c><00:28:41.440><c> another</c>"
+    },
+    {
+      "start": 1721.83,
+      "duration": 0.0,
+      "text": "now great data um so data is another"
+    },
+    {
+      "start": 1721.84,
+      "duration": 0.0,
+      "text": "now great data um so data is another really<00:28:42.080><c> big</c><00:28:42.360><c> topic</c><00:28:43.120><c> um</c><00:28:43.320><c> at</c><00:28:43.440><c> a</c><00:28:43.600><c> high</c><00:28:43.799><c> level</c>"
+    },
+    {
+      "start": 1724.35,
+      "duration": 0.0,
+      "text": "really big topic um at a high level"
+    },
+    {
+      "start": 1724.36,
+      "duration": 0.0,
+      "text": "really big topic um at a high level people<00:28:44.720><c> just</c><00:28:44.880><c> say</c><00:28:45.200><c> oh</c><00:28:45.360><c> you</c><00:28:45.519><c> basically</c><00:28:45.840><c> train</c>"
+    },
+    {
+      "start": 1726.269,
+      "duration": 0.0,
+      "text": "people just say oh you basically train"
+    },
+    {
+      "start": 1726.279,
+      "duration": 0.0,
+      "text": "people just say oh you basically train large<00:28:46.559><c> language</c><00:28:46.880><c> models</c><00:28:47.240><c> on</c><00:28:47.440><c> all</c><00:28:47.640><c> of</c><00:28:47.840><c> Internet</c>"
+    },
+    {
+      "start": 1728.63,
+      "duration": 0.0,
+      "text": "large language models on all of Internet"
+    },
+    {
+      "start": 1728.64,
+      "duration": 0.0,
+      "text": "large language models on all of Internet what<00:28:48.760><c> does</c><00:28:48.919><c> that</c><00:28:49.080><c> even</c><00:28:49.279><c> mean</c><00:28:50.200><c> um</c><00:28:50.880><c> so</c><00:28:51.240><c> or</c><00:28:51.480><c> people</c>"
+    },
+    {
+      "start": 1731.83,
+      "duration": 0.0,
+      "text": "what does that even mean um so or people"
+    },
+    {
+      "start": 1731.84,
+      "duration": 0.0,
+      "text": "what does that even mean um so or people sometimes<00:28:52.080><c> say</c><00:28:52.200><c> all</c><00:28:52.360><c> of</c><00:28:52.480><c> clean</c><00:28:52.760><c> internet</c>"
+    },
+    {
+      "start": 1733.19,
+      "duration": 0.0,
+      "text": "sometimes say all of clean internet"
+    },
+    {
+      "start": 1733.2,
+      "duration": 0.0,
+      "text": "sometimes say all of clean internet which<00:28:53.519><c> is</c><00:28:53.840><c> even</c><00:28:54.080><c> less</c><00:28:54.519><c> defined</c><00:28:55.519><c> um</c><00:28:56.200><c> so</c>"
+    },
+    {
+      "start": 1736.47,
+      "duration": 0.0,
+      "text": "which is even less defined um so"
+    },
+    {
+      "start": 1736.48,
+      "duration": 0.0,
+      "text": "which is even less defined um so internet<00:28:56.880><c> is</c><00:28:57.120><c> very</c><00:28:57.360><c> dirty</c><00:28:57.840><c> and</c><00:28:58.440><c> really</c><00:28:58.679><c> not</c>"
+    },
+    {
+      "start": 1738.909,
+      "duration": 0.0,
+      "text": "internet is very dirty and really not"
+    },
+    {
+      "start": 1738.919,
+      "duration": 0.0,
+      "text": "internet is very dirty and really not representative<00:28:59.559><c> of</c><00:28:59.720><c> what</c><00:28:59.840><c> we</c><00:28:59.960><c> want</c><00:29:00.120><c> in</c>"
+    },
+    {
+      "start": 1740.269,
+      "duration": 0.0,
+      "text": "representative of what we want in"
+    },
+    {
+      "start": 1740.279,
+      "duration": 0.0,
+      "text": "representative of what we want in practice<00:29:00.919><c> if</c><00:29:01.080><c> I</c><00:29:01.279><c> download</c><00:29:02.039><c> a</c><00:29:02.240><c> random</c><00:29:02.640><c> website</c>"
+    },
+    {
+      "start": 1743.19,
+      "duration": 0.0,
+      "text": "practice if I download a random website"
+    },
+    {
+      "start": 1743.2,
+      "duration": 0.0,
+      "text": "practice if I download a random website right<00:29:03.360><c> now</c><00:29:04.120><c> you</c><00:29:04.200><c> would</c><00:29:04.360><c> be</c><00:29:04.480><c> shocked</c><00:29:04.919><c> at</c><00:29:05.120><c> what</c>"
+    },
+    {
+      "start": 1745.35,
+      "duration": 0.0,
+      "text": "right now you would be shocked at what"
+    },
+    {
+      "start": 1745.36,
+      "duration": 0.0,
+      "text": "right now you would be shocked at what is<00:29:05.559><c> in</c><00:29:05.799><c> there</c><00:29:06.039><c> it's</c><00:29:06.200><c> definitely</c><00:29:06.519><c> not</c><00:29:06.640><c> your</c>"
+    },
+    {
+      "start": 1747.19,
+      "duration": 0.0,
+      "text": "is in there it's definitely not your"
+    },
+    {
+      "start": 1747.2,
+      "duration": 0.0,
+      "text": "is in there it's definitely not your Wikipedia<00:29:08.200><c> um</c><00:29:09.080><c> so</c><00:29:10.080><c> I'll</c><00:29:10.720><c> go</c><00:29:11.000><c> really</c><00:29:11.360><c> briefly</c>"
+    },
+    {
+      "start": 1752.029,
+      "duration": 0.0,
+      "text": "Wikipedia um so I'll go really briefly"
+    },
+    {
+      "start": 1752.039,
+      "duration": 0.0,
+      "text": "Wikipedia um so I'll go really briefly on<00:29:12.279><c> like</c><00:29:12.440><c> what</c><00:29:12.600><c> people</c><00:29:12.880><c> do</c><00:29:13.640><c> um</c><00:29:14.200><c> I</c><00:29:14.279><c> can</c><00:29:14.440><c> answer</c>"
+    },
+    {
+      "start": 1754.669,
+      "duration": 0.0,
+      "text": "on like what people do um I can answer"
+    },
+    {
+      "start": 1754.679,
+      "duration": 0.0,
+      "text": "on like what people do um I can answer some<00:29:14.919><c> questions</c><00:29:15.360><c> but</c><00:29:16.000><c> I</c><00:29:16.080><c> mean</c><00:29:16.399><c> data</c><00:29:16.679><c> is</c><00:29:16.799><c> on</c><00:29:16.960><c> its</c>"
+    },
+    {
+      "start": 1757.07,
+      "duration": 0.0,
+      "text": "some questions but I mean data is on its"
+    },
+    {
+      "start": 1757.08,
+      "duration": 0.0,
+      "text": "some questions but I mean data is on its own<00:29:17.279><c> is</c><00:29:17.399><c> a</c><00:29:17.640><c> huge</c><00:29:18.200><c> topic</c><00:29:19.200><c> uh</c><00:29:19.399><c> basically</c><00:29:19.880><c> first</c>"
+    },
+    {
+      "start": 1760.149,
+      "duration": 0.0,
+      "text": "own is a huge topic uh basically first"
+    },
+    {
+      "start": 1760.159,
+      "duration": 0.0,
+      "text": "own is a huge topic uh basically first what<00:29:20.279><c> you</c><00:29:20.440><c> do</c><00:29:20.720><c> is</c><00:29:21.000><c> download</c><00:29:21.440><c> all</c><00:29:21.640><c> of</c><00:29:21.799><c> Internet</c>"
+    },
+    {
+      "start": 1762.59,
+      "duration": 0.0,
+      "text": "what you do is download all of Internet"
+    },
+    {
+      "start": 1762.6,
+      "duration": 0.0,
+      "text": "what you do is download all of Internet what<00:29:22.760><c> that</c><00:29:22.919><c> means</c><00:29:23.320><c> is</c><00:29:23.440><c> that</c><00:29:23.559><c> you</c><00:29:23.679><c> use</c><00:29:24.360><c> uh</c><00:29:24.679><c> web</c>"
+    },
+    {
+      "start": 1764.95,
+      "duration": 0.0,
+      "text": "what that means is that you use uh web"
+    },
+    {
+      "start": 1764.96,
+      "duration": 0.0,
+      "text": "what that means is that you use uh web crowlers<00:29:25.840><c> that</c><00:29:25.960><c> will</c><00:29:26.200><c> go</c><00:29:26.399><c> on</c><00:29:26.640><c> every</c><00:29:26.919><c> web</c><00:29:27.159><c> page</c>"
+    },
+    {
+      "start": 1767.31,
+      "duration": 0.0,
+      "text": "crowlers that will go on every web page"
+    },
+    {
+      "start": 1767.32,
+      "duration": 0.0,
+      "text": "crowlers that will go on every web page on<00:29:27.519><c> Internet</c><00:29:28.120><c> or</c><00:29:28.320><c> every</c><00:29:28.519><c> web</c><00:29:28.760><c> page</c><00:29:29.000><c> that</c><00:29:29.159><c> is</c><00:29:30.000><c> um</c>"
+    },
+    {
+      "start": 1770.43,
+      "duration": 0.0,
+      "text": "on Internet or every web page that is um"
+    },
+    {
+      "start": 1770.44,
+      "duration": 0.0,
+      "text": "on Internet or every web page that is um on<00:29:30.840><c> Google</c><00:29:31.799><c> uh</c><00:29:32.000><c> and</c><00:29:32.240><c> that</c><00:29:32.360><c> is</c><00:29:32.600><c> around</c><00:29:33.279><c> 250</c>"
+    },
+    {
+      "start": 1774.23,
+      "duration": 0.0,
+      "text": "on Google uh and that is around 250"
+    },
+    {
+      "start": 1774.24,
+      "duration": 0.0,
+      "text": "on Google uh and that is around 250 billion<00:29:34.600><c> pages</c><00:29:35.000><c> right</c><00:29:35.159><c> now</c><00:29:35.679><c> um</c><00:29:36.519><c> and</c><00:29:36.760><c> that's</c>"
+    },
+    {
+      "start": 1776.95,
+      "duration": 0.0,
+      "text": "billion pages right now um and that's"
+    },
+    {
+      "start": 1776.96,
+      "duration": 0.0,
+      "text": "billion pages right now um and that's around<00:29:37.200><c> one</c><00:29:37.440><c> petabyte</c><00:29:38.279><c> of</c><00:29:38.640><c> of</c><00:29:38.840><c> data</c><00:29:39.399><c> so</c><00:29:39.640><c> this</c>"
+    },
+    {
+      "start": 1779.75,
+      "duration": 0.0,
+      "text": "around one petabyte of of data so this"
+    },
+    {
+      "start": 1779.76,
+      "duration": 0.0,
+      "text": "around one petabyte of of data so this is<00:29:40.039><c> actually</c><00:29:40.440><c> a</c><00:29:40.640><c> common</c><00:29:41.080><c> common</c><00:29:41.399><c> C</c><00:29:41.840><c> is</c><00:29:42.000><c> one</c><00:29:42.240><c> web</c>"
+    },
+    {
+      "start": 1782.47,
+      "duration": 0.0,
+      "text": "is actually a common common C is one web"
+    },
+    {
+      "start": 1782.48,
+      "duration": 0.0,
+      "text": "is actually a common common C is one web crowler<00:29:42.880><c> so</c><00:29:43.039><c> people</c><00:29:43.240><c> will</c><00:29:43.399><c> usually</c><00:29:43.640><c> write</c>"
+    },
+    {
+      "start": 1783.87,
+      "duration": 0.0,
+      "text": "crowler so people will usually write"
+    },
+    {
+      "start": 1783.88,
+      "duration": 0.0,
+      "text": "crowler so people will usually write their<00:29:44.080><c> own</c><00:29:44.279><c> web</c><00:29:44.440><c> crowlers</c><00:29:45.039><c> what</c><00:29:45.159><c> they</c><00:29:45.279><c> do</c><00:29:45.440><c> is</c>"
+    },
+    {
+      "start": 1785.549,
+      "duration": 0.0,
+      "text": "their own web crowlers what they do is"
+    },
+    {
+      "start": 1785.559,
+      "duration": 0.0,
+      "text": "their own web crowlers what they do is that<00:29:45.720><c> they</c><00:29:45.840><c> use</c><00:29:46.440><c> standard</c><00:29:46.880><c> web</c><00:29:47.120><c> crowlers</c><00:29:47.600><c> and</c>"
+    },
+    {
+      "start": 1787.83,
+      "duration": 0.0,
+      "text": "that they use standard web crowlers and"
+    },
+    {
+      "start": 1787.84,
+      "duration": 0.0,
+      "text": "that they use standard web crowlers and we<00:29:48.039><c> common</c><00:29:48.320><c> crawl</c><00:29:48.760><c> is</c><00:29:48.919><c> one</c><00:29:49.080><c> of</c><00:29:49.240><c> them</c><00:29:49.960><c> uh</c><00:29:50.120><c> that</c>"
+    },
+    {
+      "start": 1790.269,
+      "duration": 0.0,
+      "text": "we common crawl is one of them uh that"
+    },
+    {
+      "start": 1790.279,
+      "duration": 0.0,
+      "text": "we common crawl is one of them uh that basically<00:29:50.679><c> every</c><00:29:50.919><c> month</c><00:29:51.559><c> adds</c><00:29:51.840><c> all</c><00:29:52.159><c> the</c><00:29:52.320><c> new</c>"
+    },
+    {
+      "start": 1792.75,
+      "duration": 0.0,
+      "text": "basically every month adds all the new"
+    },
+    {
+      "start": 1792.76,
+      "duration": 0.0,
+      "text": "basically every month adds all the new websites<00:29:53.519><c> that</c><00:29:53.679><c> were</c><00:29:53.919><c> added</c><00:29:54.679><c> on</c><00:29:55.159><c> uh</c><00:29:55.320><c> internet</c>"
+    },
+    {
+      "start": 1795.669,
+      "duration": 0.0,
+      "text": "websites that were added on uh internet"
+    },
+    {
+      "start": 1795.679,
+      "duration": 0.0,
+      "text": "websites that were added on uh internet that<00:29:55.799><c> are</c><00:29:55.960><c> found</c><00:29:56.279><c> by</c><00:29:56.519><c> by</c><00:29:56.679><c> Google</c><00:29:57.240><c> and</c><00:29:57.360><c> they</c><00:29:57.519><c> put</c>"
+    },
+    {
+      "start": 1797.63,
+      "duration": 0.0,
+      "text": "that are found by by Google and they put"
+    },
+    {
+      "start": 1797.64,
+      "duration": 0.0,
+      "text": "that are found by by Google and they put it<00:29:57.720><c> in</c><00:29:57.799><c> a</c><00:29:58.120><c> big</c><00:29:58.720><c> uh</c><00:29:58.840><c> basically</c><00:29:59.159><c> a</c><00:29:59.279><c> big</c><00:29:59.480><c> data</c><00:29:59.760><c> set</c>"
+    },
+    {
+      "start": 1800.549,
+      "duration": 0.0,
+      "text": "it in a big uh basically a big data set"
+    },
+    {
+      "start": 1800.559,
+      "duration": 0.0,
+      "text": "it in a big uh basically a big data set um<00:30:00.880><c> so</c><00:30:01.480><c> that's</c><00:30:01.640><c> on</c><00:30:01.840><c> common</c><00:30:02.120><c> call</c><00:30:02.360><c> you</c><00:30:02.440><c> have</c>"
+    },
+    {
+      "start": 1802.59,
+      "duration": 0.0,
+      "text": "um so that's on common call you have"
+    },
+    {
+      "start": 1802.6,
+      "duration": 0.0,
+      "text": "um so that's on common call you have around<00:30:02.799><c> 250</c><00:30:03.440><c> billion</c><00:30:03.760><c> pages</c><00:30:04.159><c> right</c><00:30:04.279><c> now</c><00:30:04.559><c> so</c><00:30:04.960><c> 1</c>"
+    },
+    {
+      "start": 1805.269,
+      "duration": 0.0,
+      "text": "around 250 billion pages right now so 1"
+    },
+    {
+      "start": 1805.279,
+      "duration": 0.0,
+      "text": "around 250 billion pages right now so 1 E6<00:30:05.799><c> gigabytes</c><00:30:06.760><c> of</c><00:30:07.120><c> data</c><00:30:08.120><c> once</c><00:30:08.279><c> you</c><00:30:08.399><c> have</c><00:30:08.640><c> this</c>"
+    },
+    {
+      "start": 1809.389,
+      "duration": 0.0,
+      "text": "E6 gigabytes of data once you have this"
+    },
+    {
+      "start": 1809.399,
+      "duration": 0.0,
+      "text": "E6 gigabytes of data once you have this uh<00:30:09.519><c> so</c><00:30:09.679><c> this</c><00:30:09.760><c> is</c><00:30:09.880><c> a</c><00:30:10.080><c> random</c><00:30:10.679><c> web</c><00:30:10.919><c> page</c><00:30:11.519><c> like</c>"
+    },
+    {
+      "start": 1811.669,
+      "duration": 0.0,
+      "text": "uh so this is a random web page like"
+    },
+    {
+      "start": 1811.679,
+      "duration": 0.0,
+      "text": "uh so this is a random web page like literally<00:30:12.120><c> random</c><00:30:13.000><c> uh</c><00:30:13.120><c> from</c><00:30:13.360><c> this</c><00:30:13.519><c> common</c>"
+    },
+    {
+      "start": 1813.789,
+      "duration": 0.0,
+      "text": "literally random uh from this common"
+    },
+    {
+      "start": 1813.799,
+      "duration": 0.0,
+      "text": "literally random uh from this common craw<00:30:14.519><c> and</c><00:30:14.679><c> what</c><00:30:14.799><c> you</c><00:30:14.919><c> see</c><00:30:15.159><c> is</c><00:30:15.279><c> that</c><00:30:15.519><c> one</c><00:30:15.720><c> it</c>"
+    },
+    {
+      "start": 1815.87,
+      "duration": 0.0,
+      "text": "craw and what you see is that one it"
+    },
+    {
+      "start": 1815.88,
+      "duration": 0.0,
+      "text": "craw and what you see is that one it really<00:30:16.039><c> doesn't</c><00:30:16.320><c> look</c><00:30:16.480><c> at</c><00:30:17.200><c> type</c><00:30:17.360><c> of</c><00:30:17.480><c> things</c>"
+    },
+    {
+      "start": 1817.669,
+      "duration": 0.0,
+      "text": "really doesn't look at type of things"
+    },
+    {
+      "start": 1817.679,
+      "duration": 0.0,
+      "text": "really doesn't look at type of things that<00:30:17.799><c> you</c><00:30:17.919><c> would</c><00:30:18.200><c> usually</c><00:30:18.519><c> see</c><00:30:18.840><c> but</c><00:30:19.039><c> actually</c>"
+    },
+    {
+      "start": 1819.47,
+      "duration": 0.0,
+      "text": "that you would usually see but actually"
+    },
+    {
+      "start": 1819.48,
+      "duration": 0.0,
+      "text": "that you would usually see but actually so<00:30:19.679><c> this</c><00:30:19.760><c> is</c><00:30:19.840><c> an</c><00:30:20.000><c> HTML</c><00:30:20.640><c> page</c><00:30:21.480><c> uh</c><00:30:21.600><c> it's</c><00:30:21.799><c> hard</c><00:30:22.080><c> to</c>"
+    },
+    {
+      "start": 1822.269,
+      "duration": 0.0,
+      "text": "so this is an HTML page uh it's hard to"
+    },
+    {
+      "start": 1822.279,
+      "duration": 0.0,
+      "text": "so this is an HTML page uh it's hard to see<00:30:22.720><c> but</c><00:30:23.200><c> if</c><00:30:23.360><c> you</c><00:30:23.679><c> look</c><00:30:24.000><c> through</c><00:30:24.720><c> you</c><00:30:24.840><c> will</c><00:30:25.039><c> see</c>"
+    },
+    {
+      "start": 1825.389,
+      "duration": 0.0,
+      "text": "see but if you look through you will see"
+    },
+    {
+      "start": 1825.399,
+      "duration": 0.0,
+      "text": "see but if you look through you will see some<00:30:25.799><c> content</c><00:30:26.399><c> for</c><00:30:26.679><c> example</c><00:30:27.679><c> here</c><00:30:28.039><c> here</c><00:30:29.039><c> uh</c>"
+    },
+    {
+      "start": 1829.47,
+      "duration": 0.0,
+      "text": "some content for example here here uh"
+    },
+    {
+      "start": 1829.48,
+      "duration": 0.0,
+      "text": "some content for example here here uh tesing<00:30:30.120><c> world</c><00:30:30.720><c> is</c><00:30:30.840><c> your</c><00:30:31.080><c> ultimate</c><00:30:31.640><c> source</c><00:30:32.039><c> for</c>"
+    },
+    {
+      "start": 1832.23,
+      "duration": 0.0,
+      "text": "tesing world is your ultimate source for"
+    },
+    {
+      "start": 1832.24,
+      "duration": 0.0,
+      "text": "tesing world is your ultimate source for the<00:30:32.440><c> system</c><00:30:32.840><c> X</c><00:30:33.200><c> high</c><00:30:33.440><c> performance</c><00:30:33.919><c> server</c><00:30:34.240><c> and</c>"
+    },
+    {
+      "start": 1834.35,
+      "duration": 0.0,
+      "text": "the system X high performance server and"
+    },
+    {
+      "start": 1834.36,
+      "duration": 0.0,
+      "text": "the system X high performance server and then<00:30:34.480><c> you</c><00:30:34.559><c> have</c><00:30:34.760><c> three</c><00:30:35.000><c> dots</c><00:30:35.279><c> so</c><00:30:35.399><c> you</c><00:30:35.480><c> don't</c>"
+    },
+    {
+      "start": 1835.63,
+      "duration": 0.0,
+      "text": "then you have three dots so you don't"
+    },
+    {
+      "start": 1835.64,
+      "duration": 0.0,
+      "text": "then you have three dots so you don't even<00:30:36.080><c> the</c><00:30:36.159><c> sentence</c><00:30:36.480><c> is</c><00:30:36.600><c> not</c><00:30:36.760><c> even</c><00:30:37.000><c> finished</c>"
+    },
+    {
+      "start": 1837.83,
+      "duration": 0.0,
+      "text": "even the sentence is not even finished"
+    },
+    {
+      "start": 1837.84,
+      "duration": 0.0,
+      "text": "even the sentence is not even finished that's<00:30:38.080><c> how</c><00:30:38.720><c> a</c><00:30:38.880><c> random</c><00:30:39.200><c> internet</c><00:30:39.559><c> looks</c><00:30:39.919><c> like</c>"
+    },
+    {
+      "start": 1840.909,
+      "duration": 0.0,
+      "text": "that's how a random internet looks like"
+    },
+    {
+      "start": 1840.919,
+      "duration": 0.0,
+      "text": "that's how a random internet looks like uh<00:30:41.000><c> so</c><00:30:41.159><c> of</c><00:30:41.320><c> course</c><00:30:41.600><c> it's</c><00:30:41.760><c> not</c><00:30:41.960><c> that</c><00:30:42.120><c> useful</c><00:30:42.559><c> if</c>"
+    },
+    {
+      "start": 1842.63,
+      "duration": 0.0,
+      "text": "uh so of course it's not that useful if"
+    },
+    {
+      "start": 1842.64,
+      "duration": 0.0,
+      "text": "uh so of course it's not that useful if you<00:30:42.799><c> just</c><00:30:43.000><c> train</c><00:30:43.320><c> a</c><00:30:43.519><c> like</c><00:30:43.679><c> large</c><00:30:43.960><c> language</c>"
+    },
+    {
+      "start": 1844.19,
+      "duration": 0.0,
+      "text": "you just train a like large language"
+    },
+    {
+      "start": 1844.2,
+      "duration": 0.0,
+      "text": "you just train a like large language model<00:30:44.440><c> to</c><00:30:44.559><c> generate</c><00:30:44.919><c> things</c><00:30:45.159><c> like</c><00:30:45.320><c> this</c><00:30:46.000><c> so</c>"
+    },
+    {
+      "start": 1846.19,
+      "duration": 0.0,
+      "text": "model to generate things like this so"
+    },
+    {
+      "start": 1846.2,
+      "duration": 0.0,
+      "text": "model to generate things like this so what<00:30:46.279><c> are</c><00:30:46.399><c> some</c><00:30:46.559><c> of</c><00:30:46.640><c> the</c><00:30:46.760><c> steps</c><00:30:46.960><c> that</c><00:30:47.039><c> are</c>"
+    },
+    {
+      "start": 1847.35,
+      "duration": 0.0,
+      "text": "what are some of the steps that are"
+    },
+    {
+      "start": 1847.36,
+      "duration": 0.0,
+      "text": "what are some of the steps that are needed<00:30:48.360><c> first</c><00:30:48.600><c> one</c><00:30:49.039><c> you</c><00:30:49.440><c> extract</c><00:30:49.840><c> the</c><00:30:50.000><c> text</c>"
+    },
+    {
+      "start": 1850.509,
+      "duration": 0.0,
+      "text": "needed first one you extract the text"
+    },
+    {
+      "start": 1850.519,
+      "duration": 0.0,
+      "text": "needed first one you extract the text from<00:30:50.720><c> the</c><00:30:50.840><c> HTML</c><00:30:51.360><c> so</c><00:30:51.480><c> that's</c><00:30:51.600><c> what</c><00:30:51.720><c> I</c><00:30:51.840><c> just</c><00:30:52.000><c> try</c>"
+    },
+    {
+      "start": 1852.19,
+      "duration": 0.0,
+      "text": "from the HTML so that's what I just try"
+    },
+    {
+      "start": 1852.2,
+      "duration": 0.0,
+      "text": "from the HTML so that's what I just try to<00:30:52.320><c> do</c><00:30:52.519><c> by</c><00:30:52.679><c> looking</c><00:30:53.000><c> at</c><00:30:53.519><c> uh</c><00:30:53.640><c> basically</c><00:30:53.960><c> the</c>"
+    },
+    {
+      "start": 1854.07,
+      "duration": 0.0,
+      "text": "to do by looking at uh basically the"
+    },
+    {
+      "start": 1854.08,
+      "duration": 0.0,
+      "text": "to do by looking at uh basically the correct<00:30:54.399><c> text</c><00:30:55.360><c> uh</c><00:30:55.559><c> there</c><00:30:55.679><c> are</c><00:30:55.760><c> a</c><00:30:55.880><c> lot</c><00:30:56.000><c> of</c>"
+    },
+    {
+      "start": 1856.149,
+      "duration": 0.0,
+      "text": "correct text uh there are a lot of"
+    },
+    {
+      "start": 1856.159,
+      "duration": 0.0,
+      "text": "correct text uh there are a lot of challenges<00:30:56.720><c> by</c><00:30:57.080><c> through</c><00:30:57.320><c> this</c><00:30:57.480><c> for</c><00:30:57.600><c> example</c>"
+    },
+    {
+      "start": 1858.029,
+      "duration": 0.0,
+      "text": "challenges by through this for example"
+    },
+    {
+      "start": 1858.039,
+      "duration": 0.0,
+      "text": "challenges by through this for example extracting<00:30:58.519><c> math</c><00:30:59.159><c> is</c><00:30:59.360><c> actually</c><00:30:59.679><c> very</c>"
+    },
+    {
+      "start": 1860.07,
+      "duration": 0.0,
+      "text": "extracting math is actually very"
+    },
+    {
+      "start": 1860.08,
+      "duration": 0.0,
+      "text": "extracting math is actually very complicated<00:31:01.080><c> but</c><00:31:01.279><c> pretty</c><00:31:01.519><c> important</c><00:31:01.799><c> for</c>"
+    },
+    {
+      "start": 1861.909,
+      "duration": 0.0,
+      "text": "complicated but pretty important for"
+    },
+    {
+      "start": 1861.919,
+      "duration": 0.0,
+      "text": "complicated but pretty important for training<00:31:02.200><c> large</c><00:31:02.440><c> language</c><00:31:02.799><c> models</c><00:31:03.679><c> um</c><00:31:03.960><c> or</c><00:31:04.240><c> for</c>"
+    },
+    {
+      "start": 1864.389,
+      "duration": 0.0,
+      "text": "training large language models um or for"
+    },
+    {
+      "start": 1864.399,
+      "duration": 0.0,
+      "text": "training large language models um or for example<00:31:04.679><c> boiler</c><00:31:05.080><c> plates</c><00:31:05.559><c> a</c><00:31:05.639><c> lot</c><00:31:05.799><c> of</c><00:31:05.960><c> your</c>"
+    },
+    {
+      "start": 1866.07,
+      "duration": 0.0,
+      "text": "example boiler plates a lot of your"
+    },
+    {
+      "start": 1866.08,
+      "duration": 0.0,
+      "text": "example boiler plates a lot of your forums<00:31:06.760><c> will</c><00:31:06.960><c> have</c><00:31:07.159><c> the</c><00:31:07.279><c> same</c><00:31:07.519><c> type</c><00:31:07.679><c> of</c>"
+    },
+    {
+      "start": 1867.83,
+      "duration": 0.0,
+      "text": "forums will have the same type of"
+    },
+    {
+      "start": 1867.84,
+      "duration": 0.0,
+      "text": "forums will have the same type of headers<00:31:08.240><c> the</c><00:31:08.360><c> same</c><00:31:08.639><c> type</c><00:31:08.840><c> of</c><00:31:09.159><c> Footers</c><00:31:10.080><c> uh</c><00:31:10.159><c> you</c>"
+    },
+    {
+      "start": 1870.23,
+      "duration": 0.0,
+      "text": "headers the same type of Footers uh you"
+    },
+    {
+      "start": 1870.24,
+      "duration": 0.0,
+      "text": "headers the same type of Footers uh you don't<00:31:10.399><c> want</c><00:31:10.519><c> to</c><00:31:10.679><c> repeat</c><00:31:11.039><c> all</c><00:31:11.120><c> of</c><00:31:11.279><c> this</c><00:31:11.399><c> in</c><00:31:11.519><c> your</c>"
+    },
+    {
+      "start": 1872.43,
+      "duration": 0.0,
+      "text": "don't want to repeat all of this in your"
+    },
+    {
+      "start": 1872.44,
+      "duration": 0.0,
+      "text": "don't want to repeat all of this in your data<00:31:13.440><c> um</c><00:31:14.000><c> then</c><00:31:14.159><c> you</c><00:31:14.240><c> will</c><00:31:14.480><c> filter</c><00:31:14.919><c> undesirable</c>"
+    },
+    {
+      "start": 1875.71,
+      "duration": 0.0,
+      "text": "data um then you will filter undesirable"
+    },
+    {
+      "start": 1875.72,
+      "duration": 0.0,
+      "text": "data um then you will filter undesirable content<00:31:16.720><c> uh</c><00:31:16.880><c> so</c><00:31:17.200><c> not</c><00:31:17.480><c> safe</c><00:31:17.760><c> for</c><00:31:18.000><c> work</c><00:31:18.559><c> harmful</c>"
+    },
+    {
+      "start": 1879.07,
+      "duration": 0.0,
+      "text": "content uh so not safe for work harmful"
+    },
+    {
+      "start": 1879.08,
+      "duration": 0.0,
+      "text": "content uh so not safe for work harmful content<00:31:19.519><c> pii</c><00:31:20.519><c> uh</c><00:31:20.600><c> so</c><00:31:20.760><c> usually</c><00:31:21.120><c> every</c><00:31:21.320><c> company</c>"
+    },
+    {
+      "start": 1881.95,
+      "duration": 0.0,
+      "text": "content pii uh so usually every company"
+    },
+    {
+      "start": 1881.96,
+      "duration": 0.0,
+      "text": "content pii uh so usually every company has<00:31:22.159><c> basically</c><00:31:22.639><c> a</c><00:31:23.120><c> a</c><00:31:23.760><c> black</c><00:31:24.200><c> list</c><00:31:24.960><c> of</c><00:31:25.200><c> websites</c>"
+    },
+    {
+      "start": 1885.789,
+      "duration": 0.0,
+      "text": "has basically a a black list of websites"
+    },
+    {
+      "start": 1885.799,
+      "duration": 0.0,
+      "text": "has basically a a black list of websites that<00:31:25.919><c> they</c><00:31:26.039><c> don't</c><00:31:26.200><c> want</c><00:31:26.279><c> to</c><00:31:26.399><c> train</c><00:31:26.720><c> the</c><00:31:26.840><c> models</c>"
+    },
+    {
+      "start": 1887.19,
+      "duration": 0.0,
+      "text": "that they don't want to train the models"
+    },
+    {
+      "start": 1887.2,
+      "duration": 0.0,
+      "text": "that they don't want to train the models on<00:31:27.600><c> that</c><00:31:27.919><c> Black</c><00:31:28.080><c> List</c><00:31:28.279><c> is</c><00:31:28.440><c> very</c><00:31:28.679><c> long</c><00:31:29.279><c> and</c><00:31:29.440><c> you</c>"
+    },
+    {
+      "start": 1889.59,
+      "duration": 0.0,
+      "text": "on that Black List is very long and you"
+    },
+    {
+      "start": 1889.6,
+      "duration": 0.0,
+      "text": "on that Black List is very long and you basically<00:31:29.919><c> say</c><00:31:30.120><c> if</c><00:31:30.240><c> it</c><00:31:30.320><c> comes</c><00:31:30.519><c> from</c><00:31:30.760><c> there</c><00:31:31.000><c> we</c>"
+    },
+    {
+      "start": 1891.11,
+      "duration": 0.0,
+      "text": "basically say if it comes from there we"
+    },
+    {
+      "start": 1891.12,
+      "duration": 0.0,
+      "text": "basically say if it comes from there we don't<00:31:31.279><c> train</c><00:31:31.519><c> on</c><00:31:31.720><c> this</c><00:31:32.080><c> there</c><00:31:32.200><c> are</c><00:31:32.360><c> other</c><00:31:32.600><c> ways</c>"
+    },
+    {
+      "start": 1892.789,
+      "duration": 0.0,
+      "text": "don't train on this there are other ways"
+    },
+    {
+      "start": 1892.799,
+      "duration": 0.0,
+      "text": "don't train on this there are other ways of<00:31:32.960><c> doing</c><00:31:33.399><c> these</c><00:31:33.559><c> things</c><00:31:33.919><c> is</c><00:31:34.039><c> that</c><00:31:34.159><c> you</c><00:31:34.240><c> can</c>"
+    },
+    {
+      "start": 1894.389,
+      "duration": 0.0,
+      "text": "of doing these things is that you can"
+    },
+    {
+      "start": 1894.399,
+      "duration": 0.0,
+      "text": "of doing these things is that you can train<00:31:34.639><c> a</c><00:31:34.799><c> small</c><00:31:35.279><c> model</c><00:31:35.880><c> for</c><00:31:36.039><c> classifying</c><00:31:36.600><c> what</c>"
+    },
+    {
+      "start": 1896.71,
+      "duration": 0.0,
+      "text": "train a small model for classifying what"
+    },
+    {
+      "start": 1896.72,
+      "duration": 0.0,
+      "text": "train a small model for classifying what is<00:31:36.880><c> pii</c><00:31:37.559><c> removing</c><00:31:38.120><c> these</c><00:31:38.320><c> things</c><00:31:39.240><c> um</c><00:31:39.799><c> it's</c>"
+    },
+    {
+      "start": 1899.99,
+      "duration": 0.0,
+      "text": "is pii removing these things um it's"
+    },
+    {
+      "start": 1900.0,
+      "duration": 0.0,
+      "text": "is pii removing these things um it's hard<00:31:40.440><c> every</c><00:31:40.799><c> Point</c><00:31:41.120><c> here</c><00:31:41.600><c> that</c><00:31:41.760><c> I'm</c><00:31:41.880><c> going</c><00:31:42.000><c> to</c>"
+    },
+    {
+      "start": 1902.11,
+      "duration": 0.0,
+      "text": "hard every Point here that I'm going to"
+    },
+    {
+      "start": 1902.12,
+      "duration": 0.0,
+      "text": "hard every Point here that I'm going to show<00:31:42.360><c> you</c><00:31:42.720><c> is</c><00:31:43.080><c> like</c><00:31:43.880><c> a</c><00:31:44.200><c> hard</c><00:31:44.919><c> amount</c><00:31:45.200><c> of</c><00:31:45.399><c> work</c>"
+    },
+    {
+      "start": 1906.19,
+      "duration": 0.0,
+      "text": "show you is like a hard amount of work"
+    },
+    {
+      "start": 1906.2,
+      "duration": 0.0,
+      "text": "show you is like a hard amount of work uh<00:31:46.399><c> but</c><00:31:46.480><c> I'm</c><00:31:46.760><c> going</c><00:31:46.880><c> to</c><00:31:47.080><c> go</c><00:31:47.279><c> go</c><00:31:47.559><c> quickly</c>"
+    },
+    {
+      "start": 1907.87,
+      "duration": 0.0,
+      "text": "uh but I'm going to go go quickly"
+    },
+    {
+      "start": 1907.88,
+      "duration": 0.0,
+      "text": "uh but I'm going to go go quickly through<00:31:48.080><c> it</c><00:31:48.279><c> so</c><00:31:48.440><c> filter</c><00:31:48.760><c> undesirable</c><00:31:49.360><c> content</c>"
+    },
+    {
+      "start": 1910.31,
+      "duration": 0.0,
+      "text": "through it so filter undesirable content"
+    },
+    {
+      "start": 1910.32,
+      "duration": 0.0,
+      "text": "through it so filter undesirable content second<00:31:50.919><c> or</c><00:31:51.240><c> fourth</c><00:31:51.960><c> is</c><00:31:52.200><c> the</c><00:31:52.360><c> dup</c><00:31:52.720><c> D</c>"
+    },
+    {
+      "start": 1913.029,
+      "duration": 0.0,
+      "text": "second or fourth is the dup D"
+    },
+    {
+      "start": 1913.039,
+      "duration": 0.0,
+      "text": "second or fourth is the dup D duplication<00:31:54.039><c> as</c><00:31:54.159><c> I</c><00:31:54.360><c> said</c><00:31:55.159><c> um</c><00:31:55.639><c> you</c><00:31:55.799><c> might</c><00:31:56.120><c> have</c>"
+    },
+    {
+      "start": 1916.389,
+      "duration": 0.0,
+      "text": "duplication as I said um you might have"
+    },
+    {
+      "start": 1916.399,
+      "duration": 0.0,
+      "text": "duplication as I said um you might have things<00:31:56.600><c> like</c><00:31:56.840><c> headers</c><00:31:57.159><c> and</c><00:31:57.279><c> Footers</c><00:31:58.039><c> in</c>"
+    },
+    {
+      "start": 1918.149,
+      "duration": 0.0,
+      "text": "things like headers and Footers in"
+    },
+    {
+      "start": 1918.159,
+      "duration": 0.0,
+      "text": "things like headers and Footers in forums<00:31:58.679><c> that</c><00:31:58.799><c> are</c><00:31:59.000><c> always</c><00:31:59.279><c> the</c><00:31:59.399><c> same</c><00:31:59.799><c> you</c><00:31:59.880><c> want</c>"
+    },
+    {
+      "start": 1919.99,
+      "duration": 0.0,
+      "text": "forums that are always the same you want"
+    },
+    {
+      "start": 1920.0,
+      "duration": 0.0,
+      "text": "forums that are always the same you want to<00:32:00.200><c> remove</c><00:32:00.600><c> that</c><00:32:01.159><c> another</c><00:32:01.440><c> thing</c><00:32:01.600><c> that</c><00:32:01.679><c> you</c>"
+    },
+    {
+      "start": 1921.83,
+      "duration": 0.0,
+      "text": "to remove that another thing that you"
+    },
+    {
+      "start": 1921.84,
+      "duration": 0.0,
+      "text": "to remove that another thing that you might<00:32:02.080><c> have</c><00:32:02.480><c> is</c><00:32:02.639><c> a</c><00:32:02.799><c> lot</c><00:32:02.960><c> of</c><00:32:03.159><c> URLs</c><00:32:03.919><c> that</c><00:32:04.039><c> are</c>"
+    },
+    {
+      "start": 1924.269,
+      "duration": 0.0,
+      "text": "might have is a lot of URLs that are"
+    },
+    {
+      "start": 1924.279,
+      "duration": 0.0,
+      "text": "might have is a lot of URLs that are different<00:32:04.840><c> but</c><00:32:05.080><c> actually</c><00:32:05.320><c> show</c><00:32:05.679><c> the</c><00:32:05.880><c> same</c>"
+    },
+    {
+      "start": 1926.629,
+      "duration": 0.0,
+      "text": "different but actually show the same"
+    },
+    {
+      "start": 1926.639,
+      "duration": 0.0,
+      "text": "different but actually show the same website<00:32:07.639><c> um</c><00:32:08.519><c> and</c><00:32:09.000><c> you</c><00:32:09.159><c> might</c><00:32:09.399><c> also</c><00:32:09.679><c> have</c><00:32:09.840><c> a</c><00:32:09.960><c> lot</c>"
+    },
+    {
+      "start": 1930.07,
+      "duration": 0.0,
+      "text": "website um and you might also have a lot"
+    },
+    {
+      "start": 1930.08,
+      "duration": 0.0,
+      "text": "website um and you might also have a lot of<00:32:10.360><c> like</c><00:32:10.720><c> U</c><00:32:11.200><c> um</c><00:32:11.679><c> paragraphs</c><00:32:12.360><c> that</c><00:32:12.519><c> come</c><00:32:12.720><c> from</c>"
+    },
+    {
+      "start": 1932.99,
+      "duration": 0.0,
+      "text": "of like U um paragraphs that come from"
+    },
+    {
+      "start": 1933.0,
+      "duration": 0.0,
+      "text": "of like U um paragraphs that come from like<00:32:13.159><c> common</c><00:32:13.480><c> books</c><00:32:14.039><c> that</c><00:32:14.159><c> are</c><00:32:14.360><c> basically</c>"
+    },
+    {
+      "start": 1934.909,
+      "duration": 0.0,
+      "text": "like common books that are basically"
+    },
+    {
+      "start": 1934.919,
+      "duration": 0.0,
+      "text": "like common books that are basically duplicated<00:32:15.919><c> a</c><00:32:16.080><c> thousand</c><00:32:16.399><c> times</c><00:32:16.600><c> or</c><00:32:16.760><c> 10,000</c>"
+    },
+    {
+      "start": 1937.23,
+      "duration": 0.0,
+      "text": "duplicated a thousand times or 10,000"
+    },
+    {
+      "start": 1937.24,
+      "duration": 0.0,
+      "text": "duplicated a thousand times or 10,000 times<00:32:17.440><c> on</c><00:32:17.639><c> internet</c><00:32:18.320><c> so</c><00:32:18.480><c> you</c><00:32:18.799><c> have</c><00:32:18.880><c> to</c>"
+    },
+    {
+      "start": 1939.149,
+      "duration": 0.0,
+      "text": "times on internet so you have to"
+    },
+    {
+      "start": 1939.159,
+      "duration": 0.0,
+      "text": "times on internet so you have to duplicate<00:32:20.159><c> also</c><00:32:20.480><c> very</c><00:32:20.720><c> challenging</c><00:32:21.720><c> uh</c>"
+    },
+    {
+      "start": 1942.029,
+      "duration": 0.0,
+      "text": "duplicate also very challenging uh"
+    },
+    {
+      "start": 1942.039,
+      "duration": 0.0,
+      "text": "duplicate also very challenging uh because<00:32:22.200><c> you</c><00:32:22.320><c> have</c><00:32:22.440><c> to</c><00:32:22.559><c> do</c><00:32:22.720><c> that</c><00:32:22.919><c> at</c><00:32:23.440><c> scale</c>"
+    },
+    {
+      "start": 1944.43,
+      "duration": 0.0,
+      "text": "because you have to do that at scale"
+    },
+    {
+      "start": 1944.44,
+      "duration": 0.0,
+      "text": "because you have to do that at scale once<00:32:24.600><c> you</c><00:32:24.720><c> do</c><00:32:25.000><c> duplication</c><00:32:26.000><c> you</c><00:32:26.120><c> will</c><00:32:26.279><c> do</c><00:32:26.399><c> some</c>"
+    },
+    {
+      "start": 1946.629,
+      "duration": 0.0,
+      "text": "once you do duplication you will do some"
+    },
+    {
+      "start": 1946.639,
+      "duration": 0.0,
+      "text": "once you do duplication you will do some heuristic<00:32:27.120><c> filtering</c><00:32:27.960><c> you</c><00:32:28.080><c> will</c><00:32:28.240><c> try</c><00:32:28.399><c> to</c>"
+    },
+    {
+      "start": 1948.549,
+      "duration": 0.0,
+      "text": "heuristic filtering you will try to"
+    },
+    {
+      "start": 1948.559,
+      "duration": 0.0,
+      "text": "heuristic filtering you will try to remove<00:32:29.399><c> low</c><00:32:29.799><c> quality</c><00:32:30.320><c> documents</c><00:32:31.320><c> uh</c><00:32:31.480><c> the</c><00:32:31.600><c> way</c>"
+    },
+    {
+      "start": 1951.75,
+      "duration": 0.0,
+      "text": "remove low quality documents uh the way"
+    },
+    {
+      "start": 1951.76,
+      "duration": 0.0,
+      "text": "remove low quality documents uh the way you<00:32:31.880><c> do</c><00:32:32.120><c> that</c><00:32:32.320><c> are</c><00:32:32.559><c> things</c><00:32:32.760><c> like</c><00:32:32.960><c> rules-based</c>"
+    },
+    {
+      "start": 1953.47,
+      "duration": 0.0,
+      "text": "you do that are things like rules-based"
+    },
+    {
+      "start": 1953.48,
+      "duration": 0.0,
+      "text": "you do that are things like rules-based um<00:32:34.279><c> filtering</c><00:32:35.159><c> for</c><00:32:35.360><c> example</c><00:32:35.720><c> if</c><00:32:35.880><c> you</c><00:32:36.000><c> see</c><00:32:36.200><c> that</c>"
+    },
+    {
+      "start": 1956.31,
+      "duration": 0.0,
+      "text": "um filtering for example if you see that"
+    },
+    {
+      "start": 1956.32,
+      "duration": 0.0,
+      "text": "um filtering for example if you see that there<00:32:36.399><c> are</c><00:32:36.519><c> some</c><00:32:36.720><c> outlier</c><00:32:37.240><c> tokens</c><00:32:37.760><c> if</c><00:32:37.840><c> the</c>"
+    },
+    {
+      "start": 1957.99,
+      "duration": 0.0,
+      "text": "there are some outlier tokens if the"
+    },
+    {
+      "start": 1958.0,
+      "duration": 0.0,
+      "text": "there are some outlier tokens if the distribution<00:32:38.480><c> of</c><00:32:38.600><c> tokens</c><00:32:38.919><c> in</c><00:32:39.039><c> the</c><00:32:39.200><c> website</c><00:32:39.760><c> is</c>"
+    },
+    {
+      "start": 1959.95,
+      "duration": 0.0,
+      "text": "distribution of tokens in the website is"
+    },
+    {
+      "start": 1959.96,
+      "duration": 0.0,
+      "text": "distribution of tokens in the website is very<00:32:40.159><c> different</c><00:32:40.440><c> than</c><00:32:40.559><c> the</c><00:32:40.679><c> usual</c>"
+    },
+    {
+      "start": 1960.95,
+      "duration": 0.0,
+      "text": "very different than the usual"
+    },
+    {
+      "start": 1960.96,
+      "duration": 0.0,
+      "text": "very different than the usual distribution<00:32:41.399><c> of</c><00:32:41.559><c> tokens</c><00:32:42.120><c> then</c><00:32:42.240><c> it's</c>"
+    },
+    {
+      "start": 1962.389,
+      "duration": 0.0,
+      "text": "distribution of tokens then it's"
+    },
+    {
+      "start": 1962.399,
+      "duration": 0.0,
+      "text": "distribution of tokens then it's probably<00:32:42.679><c> some</c><00:32:42.799><c> outlier</c><00:32:43.480><c> if</c><00:32:43.600><c> you</c><00:32:43.679><c> see</c><00:32:43.919><c> that</c>"
+    },
+    {
+      "start": 1964.07,
+      "duration": 0.0,
+      "text": "probably some outlier if you see that"
+    },
+    {
+      "start": 1964.08,
+      "duration": 0.0,
+      "text": "probably some outlier if you see that the<00:32:44.159><c> length</c><00:32:44.440><c> of</c><00:32:44.559><c> the</c><00:32:44.679><c> words</c><00:32:45.200><c> in</c><00:32:45.360><c> this</c><00:32:45.559><c> website</c>"
+    },
+    {
+      "start": 1966.149,
+      "duration": 0.0,
+      "text": "the length of the words in this website"
+    },
+    {
+      "start": 1966.159,
+      "duration": 0.0,
+      "text": "the length of the words in this website is<00:32:46.360><c> super</c><00:32:46.679><c> long</c><00:32:47.159><c> there's</c><00:32:47.399><c> something</c><00:32:47.679><c> strange</c>"
+    },
+    {
+      "start": 1968.029,
+      "duration": 0.0,
+      "text": "is super long there's something strange"
+    },
+    {
+      "start": 1968.039,
+      "duration": 0.0,
+      "text": "is super long there's something strange going<00:32:48.240><c> on</c><00:32:48.399><c> on</c><00:32:48.559><c> that</c><00:32:48.720><c> website</c><00:32:49.480><c> if</c><00:32:49.559><c> you</c><00:32:49.679><c> see</c><00:32:49.919><c> that</c>"
+    },
+    {
+      "start": 1970.149,
+      "duration": 0.0,
+      "text": "going on on that website if you see that"
+    },
+    {
+      "start": 1970.159,
+      "duration": 0.0,
+      "text": "going on on that website if you see that the<00:32:50.440><c> the</c><00:32:50.559><c> website</c><00:32:50.960><c> has</c><00:32:51.080><c> only</c><00:32:51.320><c> three</c><00:32:51.840><c> words</c>"
+    },
+    {
+      "start": 1972.83,
+      "duration": 0.0,
+      "text": "the the website has only three words"
+    },
+    {
+      "start": 1972.84,
+      "duration": 0.0,
+      "text": "the the website has only three words maybe<00:32:53.120><c> is</c><00:32:53.200><c> it</c><00:32:53.320><c> worth</c><00:32:53.519><c> training</c><00:32:53.799><c> on</c><00:32:53.919><c> it</c><00:32:54.080><c> maybe</c>"
+    },
+    {
+      "start": 1974.31,
+      "duration": 0.0,
+      "text": "maybe is it worth training on it maybe"
+    },
+    {
+      "start": 1974.32,
+      "duration": 0.0,
+      "text": "maybe is it worth training on it maybe not<00:32:54.559><c> if</c><00:32:54.679><c> it</c><00:32:54.880><c> has</c><00:32:55.159><c> like</c><00:32:55.600><c> 10</c><00:32:55.840><c> million</c><00:32:56.200><c> words</c>"
+    },
+    {
+      "start": 1976.83,
+      "duration": 0.0,
+      "text": "not if it has like 10 million words"
+    },
+    {
+      "start": 1976.84,
+      "duration": 0.0,
+      "text": "not if it has like 10 million words maybe<00:32:57.080><c> there's</c><00:32:57.279><c> something</c><00:32:57.519><c> also</c>"
+    },
+    {
+      "start": 1978.549,
+      "duration": 0.0,
+      "text": "maybe there's something also"
+    },
+    {
+      "start": 1978.559,
+      "duration": 0.0,
+      "text": "maybe there's something also wrong<00:32:58.880><c> going</c><00:32:59.080><c> on</c><00:32:59.279><c> that</c><00:32:59.440><c> page</c><00:33:00.240><c> um</c><00:33:00.440><c> so</c><00:33:00.559><c> a</c><00:33:00.639><c> lot</c><00:33:00.760><c> of</c>"
+    },
+    {
+      "start": 1980.83,
+      "duration": 0.0,
+      "text": "wrong going on that page um so a lot of"
+    },
+    {
+      "start": 1980.84,
+      "duration": 0.0,
+      "text": "wrong going on that page um so a lot of rules<00:33:01.120><c> like</c><00:33:01.320><c> this</c><00:33:01.600><c> yes</c><00:33:02.080><c> why</c><00:33:02.279><c> we</c><00:33:02.480><c> filter</c><00:33:02.919><c> out</c>"
+    },
+    {
+      "start": 1983.149,
+      "duration": 0.0,
+      "text": "rules like this yes why we filter out"
+    },
+    {
+      "start": 1983.159,
+      "duration": 0.0,
+      "text": "rules like this yes why we filter out undesirable<00:33:03.840><c> content</c><00:33:04.440><c> from</c><00:33:04.600><c> our</c><00:33:04.840><c> dat</c><00:33:05.159><c> set</c>"
+    },
+    {
+      "start": 1985.35,
+      "duration": 0.0,
+      "text": "undesirable content from our dat set"
+    },
+    {
+      "start": 1985.36,
+      "duration": 0.0,
+      "text": "undesirable content from our dat set instead<00:33:05.639><c> of</c><00:33:05.799><c> kind</c>"
+    },
+    {
+      "start": 1986.629,
+      "duration": 0.0,
+      "text": "instead of kind"
+    },
+    {
+      "start": 1986.639,
+      "duration": 0.0,
+      "text": "instead of kind of<00:33:07.639><c> putting</c><00:33:07.880><c> it</c><00:33:08.000><c> in</c><00:33:08.120><c> is</c><00:33:08.279><c> like</c><00:33:08.360><c> a</c><00:33:08.519><c> supervised</c>"
+    },
+    {
+      "start": 1989.19,
+      "duration": 0.0,
+      "text": "of putting it in is like a supervised"
+    },
+    {
+      "start": 1989.2,
+      "duration": 0.0,
+      "text": "of putting it in is like a supervised loss<00:33:10.200><c> right</c><00:33:10.480><c> like</c><00:33:10.720><c> can</c><00:33:10.840><c> we</c><00:33:11.039><c> not</c><00:33:11.240><c> just</c><00:33:11.360><c> say</c><00:33:11.679><c> like</c>"
+    },
+    {
+      "start": 1992.149,
+      "duration": 0.0,
+      "text": "loss right like can we not just say like"
+    },
+    {
+      "start": 1992.159,
+      "duration": 0.0,
+      "text": "loss right like can we not just say like you<00:33:12.320><c> know</c><00:33:12.559><c> here's</c><00:33:12.840><c> this</c><00:33:13.120><c> like</c><00:33:13.320><c> hate</c><00:33:13.559><c> speech</c>"
+    },
+    {
+      "start": 1993.909,
+      "duration": 0.0,
+      "text": "you know here's this like hate speech"
+    },
+    {
+      "start": 1993.919,
+      "duration": 0.0,
+      "text": "you know here's this like hate speech website<00:33:14.440><c> let's</c><00:33:15.200><c> actively</c><00:33:15.639><c> try</c><00:33:16.320><c> to</c><00:33:17.320><c> Let's</c>"
+    },
+    {
+      "start": 1997.549,
+      "duration": 0.0,
+      "text": "website let's actively try to Let's"
+    },
+    {
+      "start": 1997.559,
+      "duration": 0.0,
+      "text": "website let's actively try to Let's actively<00:33:17.919><c> penalize</c><00:33:18.320><c> the</c><00:33:18.679><c> for</c><00:33:18.960><c> generating</c>"
+    },
+    {
+      "start": 1999.95,
+      "duration": 0.0,
+      "text": "actively penalize the for generating"
+    },
+    {
+      "start": 1999.96,
+      "duration": 0.0,
+      "text": "actively penalize the for generating we'll<00:33:20.159><c> do</c><00:33:20.440><c> exactly</c><00:33:20.919><c> that</c><00:33:21.480><c> but</c><00:33:21.639><c> not</c><00:33:21.880><c> at</c><00:33:22.080><c> this</c>"
+    },
+    {
+      "start": 2002.23,
+      "duration": 0.0,
+      "text": "we'll do exactly that but not at this"
+    },
+    {
+      "start": 2002.24,
+      "duration": 0.0,
+      "text": "we'll do exactly that but not at this step<00:33:22.720><c> that's</c><00:33:22.960><c> where</c><00:33:23.120><c> the</c><00:33:23.440><c> posttraining</c><00:33:23.880><c> will</c>"
+    },
+    {
+      "start": 2004.19,
+      "duration": 0.0,
+      "text": "step that's where the posttraining will"
+    },
+    {
+      "start": 2004.2,
+      "duration": 0.0,
+      "text": "step that's where the posttraining will come<00:33:24.440><c> from</c><00:33:25.440><c> uh</c><00:33:25.799><c> pre-training</c><00:33:26.799><c> um</c><00:33:27.840><c> the</c><00:33:28.120><c> idea</c><00:33:28.559><c> is</c>"
+    },
+    {
+      "start": 2008.71,
+      "duration": 0.0,
+      "text": "come from uh pre-training um the idea is"
+    },
+    {
+      "start": 2008.72,
+      "duration": 0.0,
+      "text": "come from uh pre-training um the idea is just<00:33:28.880><c> to</c><00:33:29.200><c> say</c><00:33:30.200><c> I</c><00:33:30.279><c> want</c><00:33:30.399><c> to</c><00:33:30.639><c> model</c><00:33:31.279><c> kind</c><00:33:31.440><c> of</c><00:33:31.840><c> how</c>"
+    },
+    {
+      "start": 2012.029,
+      "duration": 0.0,
+      "text": "just to say I want to model kind of how"
+    },
+    {
+      "start": 2012.039,
+      "duration": 0.0,
+      "text": "just to say I want to model kind of how humans<00:33:32.440><c> speak</c><00:33:33.240><c> essentially</c><00:33:34.240><c> um</c><00:33:34.679><c> and</c><00:33:34.799><c> I</c><00:33:34.880><c> want</c>"
+    },
+    {
+      "start": 2014.99,
+      "duration": 0.0,
+      "text": "humans speak essentially um and I want"
+    },
+    {
+      "start": 2015.0,
+      "duration": 0.0,
+      "text": "humans speak essentially um and I want to<00:33:35.159><c> remove</c><00:33:35.480><c> all</c><00:33:35.679><c> these</c><00:33:35.880><c> like</c><00:33:36.039><c> headers</c><00:33:36.399><c> photos</c>"
+    },
+    {
+      "start": 2016.83,
+      "duration": 0.0,
+      "text": "to remove all these like headers photos"
+    },
+    {
+      "start": 2016.84,
+      "duration": 0.0,
+      "text": "to remove all these like headers photos and<00:33:37.039><c> and</c><00:33:37.200><c> menus</c><00:33:37.600><c> and</c><00:33:37.760><c> things</c><00:33:37.960><c> like</c><00:33:38.159><c> this</c><00:33:38.720><c> but</c>"
+    },
+    {
+      "start": 2018.83,
+      "duration": 0.0,
+      "text": "and and menus and things like this but"
+    },
+    {
+      "start": 2018.84,
+      "duration": 0.0,
+      "text": "and and menus and things like this but it's<00:33:38.919><c> a</c><00:33:39.080><c> very</c><00:33:39.240><c> good</c><00:33:39.960><c> uh</c><00:33:40.279><c> like</c><00:33:40.799><c> idea</c><00:33:41.120><c> that</c><00:33:41.200><c> you</c>"
+    },
+    {
+      "start": 2021.35,
+      "duration": 0.0,
+      "text": "it's a very good uh like idea that you"
+    },
+    {
+      "start": 2021.36,
+      "duration": 0.0,
+      "text": "it's a very good uh like idea that you just<00:33:41.519><c> had</c><00:33:41.679><c> and</c><00:33:41.919><c> that's</c><00:33:42.120><c> exactly</c><00:33:42.440><c> what</c><00:33:42.559><c> we'll</c>"
+    },
+    {
+      "start": 2022.71,
+      "duration": 0.0,
+      "text": "just had and that's exactly what we'll"
+    },
+    {
+      "start": 2022.72,
+      "duration": 0.0,
+      "text": "just had and that's exactly what we'll do"
+    },
+    {
+      "start": 2024.389,
+      "duration": 0.0,
+      "text": "do"
+    },
+    {
+      "start": 2024.399,
+      "duration": 0.0,
+      "text": "do later<00:33:45.399><c> Next</c><00:33:45.639><c> Step</c><00:33:45.960><c> modelbased</c><00:33:46.519><c> filtering</c><00:33:47.120><c> so</c>"
+    },
+    {
+      "start": 2027.269,
+      "duration": 0.0,
+      "text": "later Next Step modelbased filtering so"
+    },
+    {
+      "start": 2027.279,
+      "duration": 0.0,
+      "text": "later Next Step modelbased filtering so once<00:33:47.399><c> you</c><00:33:47.559><c> filtered</c><00:33:47.880><c> a</c><00:33:47.960><c> lot</c><00:33:48.120><c> of</c><00:33:48.279><c> data</c><00:33:48.840><c> what</c><00:33:48.960><c> you</c>"
+    },
+    {
+      "start": 2029.029,
+      "duration": 0.0,
+      "text": "once you filtered a lot of data what you"
+    },
+    {
+      "start": 2029.039,
+      "duration": 0.0,
+      "text": "once you filtered a lot of data what you will<00:33:49.240><c> do</c><00:33:49.799><c> uh</c><00:33:50.080><c> that's</c><00:33:50.279><c> actually</c><00:33:50.480><c> a</c><00:33:50.600><c> very</c><00:33:50.799><c> cute</c>"
+    },
+    {
+      "start": 2031.07,
+      "duration": 0.0,
+      "text": "will do uh that's actually a very cute"
+    },
+    {
+      "start": 2031.08,
+      "duration": 0.0,
+      "text": "will do uh that's actually a very cute trick<00:33:51.880><c> uh</c><00:33:52.000><c> you</c><00:33:52.080><c> will</c><00:33:52.279><c> take</c><00:33:52.480><c> all</c><00:33:52.639><c> of</c><00:33:52.919><c> Wikipedia</c>"
+    },
+    {
+      "start": 2033.909,
+      "duration": 0.0,
+      "text": "trick uh you will take all of Wikipedia"
+    },
+    {
+      "start": 2033.919,
+      "duration": 0.0,
+      "text": "trick uh you will take all of Wikipedia and<00:33:54.039><c> you</c><00:33:54.120><c> will</c><00:33:54.279><c> look</c><00:33:54.440><c> at</c><00:33:54.720><c> all</c><00:33:54.960><c> the</c><00:33:55.200><c> links</c><00:33:56.120><c> that</c>"
+    },
+    {
+      "start": 2036.23,
+      "duration": 0.0,
+      "text": "and you will look at all the links that"
+    },
+    {
+      "start": 2036.24,
+      "duration": 0.0,
+      "text": "and you will look at all the links that are<00:33:56.440><c> linked</c><00:33:56.760><c> through</c><00:33:57.039><c> Wikipedia</c><00:33:57.519><c> p</c>"
+    },
+    {
+      "start": 2038.669,
+      "duration": 0.0,
+      "text": "are linked through Wikipedia p"
+    },
+    {
+      "start": 2038.679,
+      "duration": 0.0,
+      "text": "are linked through Wikipedia p because<00:33:59.000><c> probably</c><00:33:59.320><c> if</c><00:33:59.440><c> something</c><00:33:59.720><c> is</c>"
+    },
+    {
+      "start": 2039.83,
+      "duration": 0.0,
+      "text": "because probably if something is"
+    },
+    {
+      "start": 2039.84,
+      "duration": 0.0,
+      "text": "because probably if something is referenced<00:34:00.240><c> by</c><00:34:00.399><c> Wikipedia</c><00:34:01.080><c> it's</c><00:34:01.279><c> probably</c>"
+    },
+    {
+      "start": 2041.549,
+      "duration": 0.0,
+      "text": "referenced by Wikipedia it's probably"
+    },
+    {
+      "start": 2041.559,
+      "duration": 0.0,
+      "text": "referenced by Wikipedia it's probably some<00:34:01.760><c> high</c><00:34:01.960><c> quality</c><00:34:02.320><c> website</c><00:34:03.240><c> and</c><00:34:03.480><c> you</c><00:34:03.559><c> will</c>"
+    },
+    {
+      "start": 2043.789,
+      "duration": 0.0,
+      "text": "some high quality website and you will"
+    },
+    {
+      "start": 2043.799,
+      "duration": 0.0,
+      "text": "some high quality website and you will train<00:34:04.080><c> a</c><00:34:04.360><c> classifier</c><00:34:05.360><c> to</c><00:34:05.639><c> predict</c><00:34:06.120><c> whether</c>"
+    },
+    {
+      "start": 2046.43,
+      "duration": 0.0,
+      "text": "train a classifier to predict whether"
+    },
+    {
+      "start": 2046.44,
+      "duration": 0.0,
+      "text": "train a classifier to predict whether something<00:34:07.240><c> comes</c><00:34:07.639><c> from</c><00:34:08.000><c> whether</c><00:34:08.240><c> a</c><00:34:08.560><c> document</c>"
+    },
+    {
+      "start": 2049.27,
+      "duration": 0.0,
+      "text": "something comes from whether a document"
+    },
+    {
+      "start": 2049.28,
+      "duration": 0.0,
+      "text": "something comes from whether a document comes<00:34:09.760><c> from</c><00:34:10.320><c> one</c><00:34:10.480><c> of</c><00:34:10.679><c> these</c><00:34:11.200><c> references</c><00:34:12.200><c> uh</c>"
+    },
+    {
+      "start": 2052.27,
+      "duration": 0.0,
+      "text": "comes from one of these references uh"
+    },
+    {
+      "start": 2052.28,
+      "duration": 0.0,
+      "text": "comes from one of these references uh from<00:34:12.520><c> Wikipedia</c><00:34:13.280><c> or</c><00:34:13.480><c> whether</c><00:34:13.679><c> it's</c><00:34:14.000><c> from</c><00:34:14.200><c> the</c>"
+    },
+    {
+      "start": 2054.349,
+      "duration": 0.0,
+      "text": "from Wikipedia or whether it's from the"
+    },
+    {
+      "start": 2054.359,
+      "duration": 0.0,
+      "text": "from Wikipedia or whether it's from the random<00:34:14.760><c> web</c><00:34:15.440><c> and</c><00:34:15.560><c> you</c><00:34:15.639><c> will</c><00:34:15.879><c> try</c><00:34:16.079><c> to</c><00:34:16.320><c> basically</c>"
+    },
+    {
+      "start": 2056.75,
+      "duration": 0.0,
+      "text": "random web and you will try to basically"
+    },
+    {
+      "start": 2056.76,
+      "duration": 0.0,
+      "text": "random web and you will try to basically say<00:34:17.240><c> I</c><00:34:17.320><c> want</c><00:34:17.599><c> more</c><00:34:17.919><c> of</c><00:34:18.200><c> the</c><00:34:18.760><c> things</c><00:34:19.040><c> that</c><00:34:19.240><c> come</c>"
+    },
+    {
+      "start": 2059.51,
+      "duration": 0.0,
+      "text": "say I want more of the things that come"
+    },
+    {
+      "start": 2059.52,
+      "duration": 0.0,
+      "text": "say I want more of the things that come from<00:34:20.040><c> Wikipedia</c><00:34:20.800><c> references</c><00:34:21.800><c> does</c><00:34:21.960><c> that</c><00:34:22.079><c> make</c>"
+    },
+    {
+      "start": 2062.75,
+      "duration": 0.0,
+      "text": "from Wikipedia references does that make"
+    },
+    {
+      "start": 2062.76,
+      "duration": 0.0,
+      "text": "from Wikipedia references does that make sense<00:34:23.760><c> so</c><00:34:23.960><c> yeah</c><00:34:24.079><c> so</c><00:34:24.200><c> you</c><00:34:24.280><c> will</c><00:34:24.440><c> train</c><00:34:24.639><c> a</c><00:34:24.960><c> a</c>"
+    },
+    {
+      "start": 2065.069,
+      "duration": 0.0,
+      "text": "sense so yeah so you will train a a"
+    },
+    {
+      "start": 2065.079,
+      "duration": 0.0,
+      "text": "sense so yeah so you will train a a machine<00:34:25.320><c> learning</c><00:34:25.960><c> uh</c><00:34:26.079><c> model</c><00:34:26.520><c> usually</c><00:34:26.960><c> also</c>"
+    },
+    {
+      "start": 2067.31,
+      "duration": 0.0,
+      "text": "machine learning uh model usually also"
+    },
+    {
+      "start": 2067.32,
+      "duration": 0.0,
+      "text": "machine learning uh model usually also very<00:34:27.480><c> simp</c><00:34:27.720><c> simple</c><00:34:27.960><c> models</c><00:34:28.399><c> because</c><00:34:28.520><c> you</c><00:34:28.639><c> need</c>"
+    },
+    {
+      "start": 2068.75,
+      "duration": 0.0,
+      "text": "very simp simple models because you need"
+    },
+    {
+      "start": 2068.76,
+      "duration": 0.0,
+      "text": "very simp simple models because you need to<00:34:28.879><c> do</c><00:34:29.079><c> that</c><00:34:29.280><c> really</c><00:34:29.520><c> at</c><00:34:29.679><c> scale</c><00:34:30.040><c> I</c><00:34:30.119><c> mean</c><00:34:30.280><c> just</c>"
+    },
+    {
+      "start": 2070.349,
+      "duration": 0.0,
+      "text": "to do that really at scale I mean just"
+    },
+    {
+      "start": 2070.359,
+      "duration": 0.0,
+      "text": "to do that really at scale I mean just think<00:34:30.599><c> about</c><00:34:30.800><c> the</c><00:34:30.919><c> 250</c><00:34:31.599><c> billion</c>"
+    },
+    {
+      "start": 2072.95,
+      "duration": 0.0,
+      "text": "think about the 250 billion"
+    },
+    {
+      "start": 2072.96,
+      "duration": 0.0,
+      "text": "think about the 250 billion Pages<00:34:33.960><c> uh</c><00:34:34.159><c> next</c><00:34:34.359><c> one</c><00:34:34.960><c> you</c><00:34:35.079><c> will</c><00:34:35.919><c> try</c><00:34:36.200><c> to</c>"
+    },
+    {
+      "start": 2076.43,
+      "duration": 0.0,
+      "text": "Pages uh next one you will try to"
+    },
+    {
+      "start": 2076.44,
+      "duration": 0.0,
+      "text": "Pages uh next one you will try to classify<00:34:36.919><c> your</c><00:34:37.119><c> data</c><00:34:37.720><c> into</c><00:34:38.079><c> different</c>"
+    },
+    {
+      "start": 2078.669,
+      "duration": 0.0,
+      "text": "classify your data into different"
+    },
+    {
+      "start": 2078.679,
+      "duration": 0.0,
+      "text": "classify your data into different different<00:34:39.440><c> um</c><00:34:40.280><c> domains</c><00:34:40.879><c> you</c><00:34:41.000><c> will</c><00:34:41.159><c> say</c><00:34:41.440><c> okay</c>"
+    },
+    {
+      "start": 2081.629,
+      "duration": 0.0,
+      "text": "different um domains you will say okay"
+    },
+    {
+      "start": 2081.639,
+      "duration": 0.0,
+      "text": "different um domains you will say okay this<00:34:41.800><c> is</c><00:34:42.359><c> entertainment</c><00:34:43.040><c> this</c><00:34:43.119><c> is</c><00:34:43.320><c> books</c><00:34:43.720><c> this</c>"
+    },
+    {
+      "start": 2083.829,
+      "duration": 0.0,
+      "text": "this is entertainment this is books this"
+    },
+    {
+      "start": 2083.839,
+      "duration": 0.0,
+      "text": "this is entertainment this is books this is<00:34:44.040><c> code</c><00:34:44.679><c> this</c><00:34:44.760><c> is</c><00:34:44.960><c> like</c><00:34:45.119><c> these</c><00:34:45.280><c> type</c><00:34:45.480><c> of</c>"
+    },
+    {
+      "start": 2085.629,
+      "duration": 0.0,
+      "text": "is code this is like these type of"
+    },
+    {
+      "start": 2085.639,
+      "duration": 0.0,
+      "text": "is code this is like these type of domains<00:34:46.440><c> and</c><00:34:46.599><c> then</c><00:34:46.720><c> you</c><00:34:46.839><c> will</c><00:34:47.079><c> try</c><00:34:47.320><c> to</c><00:34:48.000><c> either</c>"
+    },
+    {
+      "start": 2088.99,
+      "duration": 0.0,
+      "text": "domains and then you will try to either"
+    },
+    {
+      "start": 2089.0,
+      "duration": 0.0,
+      "text": "domains and then you will try to either um<00:34:49.399><c> up</c><00:34:49.760><c> or</c><00:34:50.000><c> down</c><00:34:50.359><c> weight</c><00:34:51.040><c> some</c><00:34:51.280><c> of</c><00:34:51.399><c> the</c><00:34:51.520><c> domains</c>"
+    },
+    {
+      "start": 2092.47,
+      "duration": 0.0,
+      "text": "um up or down weight some of the domains"
+    },
+    {
+      "start": 2092.48,
+      "duration": 0.0,
+      "text": "um up or down weight some of the domains uh<00:34:52.639><c> for</c><00:34:52.800><c> example</c><00:34:53.159><c> you</c><00:34:53.320><c> might</c><00:34:53.520><c> say</c><00:34:54.320><c> uh</c><00:34:54.399><c> you</c>"
+    },
+    {
+      "start": 2094.55,
+      "duration": 0.0,
+      "text": "uh for example you might say uh you"
+    },
+    {
+      "start": 2094.56,
+      "duration": 0.0,
+      "text": "uh for example you might say uh you might<00:34:54.720><c> see</c><00:34:54.960><c> that</c><00:34:55.159><c> actually</c><00:34:55.440><c> if</c><00:34:55.560><c> you</c><00:34:55.679><c> train</c>"
+    },
+    {
+      "start": 2096.03,
+      "duration": 0.0,
+      "text": "might see that actually if you train"
+    },
+    {
+      "start": 2096.04,
+      "duration": 0.0,
+      "text": "might see that actually if you train more<00:34:56.240><c> on</c><00:34:56.480><c> code</c><00:34:57.200><c> then</c><00:34:57.359><c> actually</c><00:34:57.720><c> your</c><00:34:57.839><c> model</c>"
+    },
+    {
+      "start": 2098.069,
+      "duration": 0.0,
+      "text": "more on code then actually your model"
+    },
+    {
+      "start": 2098.079,
+      "duration": 0.0,
+      "text": "more on code then actually your model becomes<00:34:58.400><c> bettered</c><00:34:58.640><c> on</c><00:34:58.760><c> reasoning</c><00:34:59.160><c> so</c><00:34:59.320><c> that's</c>"
+    },
+    {
+      "start": 2099.47,
+      "duration": 0.0,
+      "text": "becomes bettered on reasoning so that's"
+    },
+    {
+      "start": 2099.48,
+      "duration": 0.0,
+      "text": "becomes bettered on reasoning so that's something<00:34:59.760><c> that</c><00:34:59.960><c> people</c><00:35:00.720><c> usually</c><00:35:01.040><c> say</c><00:35:01.240><c> in</c><00:35:01.320><c> a</c>"
+    },
+    {
+      "start": 2101.43,
+      "duration": 0.0,
+      "text": "something that people usually say in a"
+    },
+    {
+      "start": 2101.44,
+      "duration": 0.0,
+      "text": "something that people usually say in a very<00:35:01.640><c> handwavy</c><00:35:02.200><c> way</c><00:35:02.440><c> if</c><00:35:02.560><c> you</c><00:35:02.680><c> train</c><00:35:03.200><c> your</c>"
+    },
+    {
+      "start": 2103.39,
+      "duration": 0.0,
+      "text": "very handwavy way if you train your"
+    },
+    {
+      "start": 2103.4,
+      "duration": 0.0,
+      "text": "very handwavy way if you train your model<00:35:03.680><c> more</c><00:35:04.000><c> code</c><00:35:04.280><c> actually</c><00:35:04.440><c> it</c><00:35:04.560><c> helps</c>"
+    },
+    {
+      "start": 2104.829,
+      "duration": 0.0,
+      "text": "model more code actually it helps"
+    },
+    {
+      "start": 2104.839,
+      "duration": 0.0,
+      "text": "model more code actually it helps reasoning<00:35:05.400><c> so</c><00:35:05.599><c> you</c><00:35:05.760><c> want</c><00:35:05.920><c> to</c><00:35:06.240><c> upweight</c><00:35:07.079><c> the</c>"
+    },
+    {
+      "start": 2107.23,
+      "duration": 0.0,
+      "text": "reasoning so you want to upweight the"
+    },
+    {
+      "start": 2107.24,
+      "duration": 0.0,
+      "text": "reasoning so you want to upweight the coding<00:35:08.079><c> uh</c><00:35:08.240><c> distribution</c><00:35:08.839><c> because</c><00:35:09.040><c> that</c>"
+    },
+    {
+      "start": 2109.19,
+      "duration": 0.0,
+      "text": "coding uh distribution because that"
+    },
+    {
+      "start": 2109.2,
+      "duration": 0.0,
+      "text": "coding uh distribution because that helps<00:35:09.480><c> for</c><00:35:09.720><c> General</c><00:35:10.040><c> language</c><00:35:10.400><c> modeling</c>"
+    },
+    {
+      "start": 2110.79,
+      "duration": 0.0,
+      "text": "helps for General language modeling"
+    },
+    {
+      "start": 2110.8,
+      "duration": 0.0,
+      "text": "helps for General language modeling skills<00:35:11.599><c> uh</c><00:35:11.720><c> books</c><00:35:12.079><c> is</c><00:35:12.200><c> usually</c><00:35:12.520><c> also</c><00:35:12.760><c> another</c>"
+    },
+    {
+      "start": 2113.03,
+      "duration": 0.0,
+      "text": "skills uh books is usually also another"
+    },
+    {
+      "start": 2113.04,
+      "duration": 0.0,
+      "text": "skills uh books is usually also another one<00:35:13.160><c> that</c><00:35:13.320><c> people</c><00:35:13.599><c> usually</c><00:35:14.320><c> um</c><00:35:15.280><c> upweight</c>"
+    },
+    {
+      "start": 2116.23,
+      "duration": 0.0,
+      "text": "one that people usually um upweight"
+    },
+    {
+      "start": 2116.24,
+      "duration": 0.0,
+      "text": "one that people usually um upweight entertainment<00:35:17.000><c> they</c><00:35:17.160><c> usually</c><00:35:17.720><c> downweight</c><00:35:18.720><c> uh</c>"
+    },
+    {
+      "start": 2118.829,
+      "duration": 0.0,
+      "text": "entertainment they usually downweight uh"
+    },
+    {
+      "start": 2118.839,
+      "duration": 0.0,
+      "text": "entertainment they usually downweight uh so<00:35:19.079><c> things</c><00:35:19.280><c> like</c><00:35:19.520><c> this</c><00:35:19.880><c> of</c><00:35:20.000><c> course</c><00:35:20.160><c> you</c><00:35:20.240><c> want</c>"
+    },
+    {
+      "start": 2120.349,
+      "duration": 0.0,
+      "text": "so things like this of course you want"
+    },
+    {
+      "start": 2120.359,
+      "duration": 0.0,
+      "text": "so things like this of course you want to<00:35:20.520><c> do</c><00:35:20.640><c> it</c><00:35:20.960><c> so</c><00:35:21.200><c> people</c><00:35:21.480><c> used</c><00:35:21.720><c> to</c><00:35:21.880><c> do</c><00:35:22.000><c> it</c><00:35:22.280><c> maybe</c>"
+    },
+    {
+      "start": 2123.27,
+      "duration": 0.0,
+      "text": "to do it so people used to do it maybe"
+    },
+    {
+      "start": 2123.28,
+      "duration": 0.0,
+      "text": "to do it so people used to do it maybe uh<00:35:24.160><c> kind</c><00:35:24.280><c> of</c><00:35:24.440><c> theistically</c><00:35:25.440><c> now</c><00:35:25.680><c> there's</c>"
+    },
+    {
+      "start": 2125.95,
+      "duration": 0.0,
+      "text": "uh kind of theistically now there's"
+    },
+    {
+      "start": 2125.96,
+      "duration": 0.0,
+      "text": "uh kind of theistically now there's entire<00:35:26.400><c> pipelines</c><00:35:27.040><c> that</c><00:35:27.160><c> we'll</c><00:35:27.359><c> talk</c><00:35:27.839><c> about</c>"
+    },
+    {
+      "start": 2128.23,
+      "duration": 0.0,
+      "text": "entire pipelines that we'll talk about"
+    },
+    {
+      "start": 2128.24,
+      "duration": 0.0,
+      "text": "entire pipelines that we'll talk about of<00:35:28.400><c> how</c><00:35:28.560><c> to</c><00:35:28.680><c> do</c><00:35:28.920><c> these</c><00:35:29.079><c> things</c><00:35:29.599><c> uh</c><00:35:29.800><c> slightly</c>"
+    },
+    {
+      "start": 2130.23,
+      "duration": 0.0,
+      "text": "of how to do these things uh slightly"
+    },
+    {
+      "start": 2130.24,
+      "duration": 0.0,
+      "text": "of how to do these things uh slightly more<00:35:30.800><c> um</c>"
+    },
+    {
+      "start": 2132.67,
+      "duration": 0.0,
+      "text": "more um"
+    },
+    {
+      "start": 2132.68,
+      "duration": 0.0,
+      "text": "more um automatically<00:35:33.680><c> and</c><00:35:33.880><c> then</c><00:35:34.200><c> at</c><00:35:34.320><c> the</c><00:35:34.480><c> end</c><00:35:34.680><c> of</c>"
+    },
+    {
+      "start": 2134.829,
+      "duration": 0.0,
+      "text": "automatically and then at the end of"
+    },
+    {
+      "start": 2134.839,
+      "duration": 0.0,
+      "text": "automatically and then at the end of training<00:35:35.680><c> uh</c><00:35:35.920><c> usually</c><00:35:36.680><c> train</c><00:35:37.680><c> um</c><00:35:38.079><c> after</c>"
+    },
+    {
+      "start": 2138.27,
+      "duration": 0.0,
+      "text": "training uh usually train um after"
+    },
+    {
+      "start": 2138.28,
+      "duration": 0.0,
+      "text": "training uh usually train um after training<00:35:38.640><c> on</c><00:35:38.839><c> all</c><00:35:39.000><c> of</c><00:35:39.200><c> this</c><00:35:39.480><c> data</c><00:35:39.760><c> that</c><00:35:39.880><c> we</c><00:35:40.000><c> saw</c>"
+    },
+    {
+      "start": 2140.55,
+      "duration": 0.0,
+      "text": "training on all of this data that we saw"
+    },
+    {
+      "start": 2140.56,
+      "duration": 0.0,
+      "text": "training on all of this data that we saw usually<00:35:40.839><c> train</c><00:35:41.160><c> on</c><00:35:41.400><c> very</c><00:35:41.640><c> high</c><00:35:41.839><c> quality</c><00:35:42.280><c> data</c>"
+    },
+    {
+      "start": 2142.87,
+      "duration": 0.0,
+      "text": "usually train on very high quality data"
+    },
+    {
+      "start": 2142.88,
+      "duration": 0.0,
+      "text": "usually train on very high quality data at<00:35:43.040><c> the</c><00:35:43.200><c> end</c><00:35:43.839><c> of</c><00:35:44.280><c> of</c><00:35:44.440><c> training</c><00:35:44.720><c> your</c><00:35:45.119><c> large</c>"
+    },
+    {
+      "start": 2145.349,
+      "duration": 0.0,
+      "text": "at the end of of training your large"
+    },
+    {
+      "start": 2145.359,
+      "duration": 0.0,
+      "text": "at the end of of training your large language<00:35:45.640><c> model</c><00:35:46.079><c> where</c><00:35:46.240><c> you</c><00:35:46.359><c> decrease</c><00:35:46.680><c> your</c>"
+    },
+    {
+      "start": 2146.79,
+      "duration": 0.0,
+      "text": "language model where you decrease your"
+    },
+    {
+      "start": 2146.8,
+      "duration": 0.0,
+      "text": "language model where you decrease your learning<00:35:47.119><c> rate</c><00:35:47.880><c> uh</c><00:35:48.040><c> and</c><00:35:48.240><c> that</c><00:35:48.520><c> basically</c>"
+    },
+    {
+      "start": 2148.829,
+      "duration": 0.0,
+      "text": "learning rate uh and that basically"
+    },
+    {
+      "start": 2148.839,
+      "duration": 0.0,
+      "text": "learning rate uh and that basically means<00:35:49.079><c> that</c><00:35:49.200><c> you're</c><00:35:49.440><c> kind</c><00:35:49.599><c> of</c><00:35:49.800><c> overfitting</c>"
+    },
+    {
+      "start": 2150.43,
+      "duration": 0.0,
+      "text": "means that you're kind of overfitting"
+    },
+    {
+      "start": 2150.44,
+      "duration": 0.0,
+      "text": "means that you're kind of overfitting your<00:35:50.599><c> model</c><00:35:51.160><c> on</c><00:35:51.280><c> a</c><00:35:51.480><c> very</c><00:35:51.800><c> high</c><00:35:52.000><c> quality</c><00:35:52.440><c> data</c>"
+    },
+    {
+      "start": 2152.87,
+      "duration": 0.0,
+      "text": "your model on a very high quality data"
+    },
+    {
+      "start": 2152.88,
+      "duration": 0.0,
+      "text": "your model on a very high quality data so<00:35:53.040><c> usually</c><00:35:53.319><c> what</c><00:35:53.440><c> you</c><00:35:53.560><c> do</c><00:35:53.839><c> there</c><00:35:54.160><c> is</c><00:35:54.280><c> like</c>"
+    },
+    {
+      "start": 2154.47,
+      "duration": 0.0,
+      "text": "so usually what you do there is like"
+    },
+    {
+      "start": 2154.48,
+      "duration": 0.0,
+      "text": "so usually what you do there is like Wikipedia<00:35:55.480><c> you</c><00:35:56.079><c> basically</c><00:35:56.640><c> overfit</c><00:35:57.079><c> on</c>"
+    },
+    {
+      "start": 2157.19,
+      "duration": 0.0,
+      "text": "Wikipedia you basically overfit on"
+    },
+    {
+      "start": 2157.2,
+      "duration": 0.0,
+      "text": "Wikipedia you basically overfit on Wikipedia<00:35:57.760><c> yeah</c><00:35:58.119><c> and</c><00:35:58.280><c> you</c><00:35:58.480><c> overfit</c><00:35:59.400><c> on</c><00:35:59.800><c> like</c>"
+    },
+    {
+      "start": 2160.589,
+      "duration": 0.0,
+      "text": "Wikipedia yeah and you overfit on like"
+    },
+    {
+      "start": 2160.599,
+      "duration": 0.0,
+      "text": "Wikipedia yeah and you overfit on like human<00:36:01.319><c> uh</c><00:36:02.040><c> data</c><00:36:02.359><c> that</c><00:36:02.480><c> was</c><00:36:03.000><c> collected</c><00:36:04.000><c> um</c><00:36:04.480><c> the</c>"
+    },
+    {
+      "start": 2164.71,
+      "duration": 0.0,
+      "text": "human uh data that was collected um the"
+    },
+    {
+      "start": 2164.72,
+      "duration": 0.0,
+      "text": "human uh data that was collected um the other<00:36:04.960><c> things</c><00:36:05.280><c> like</c><00:36:05.440><c> continual</c><00:36:05.920><c> pre-training</c>"
+    },
+    {
+      "start": 2166.349,
+      "duration": 0.0,
+      "text": "other things like continual pre-training"
+    },
+    {
+      "start": 2166.359,
+      "duration": 0.0,
+      "text": "other things like continual pre-training for<00:36:06.560><c> getting</c><00:36:06.800><c> longer</c><00:36:07.240><c> context</c><00:36:07.960><c> I'm</c><00:36:08.119><c> I'm</c><00:36:08.200><c> going</c>"
+    },
+    {
+      "start": 2168.309,
+      "duration": 0.0,
+      "text": "for getting longer context I'm I'm going"
+    },
+    {
+      "start": 2168.319,
+      "duration": 0.0,
+      "text": "for getting longer context I'm I'm going to<00:36:08.400><c> skip</c><00:36:08.640><c> over</c><00:36:08.880><c> all</c><00:36:09.000><c> of</c><00:36:09.160><c> these</c><00:36:09.359><c> things</c><00:36:10.079><c> uh</c><00:36:10.160><c> but</c>"
+    },
+    {
+      "start": 2170.27,
+      "duration": 0.0,
+      "text": "to skip over all of these things uh but"
+    },
+    {
+      "start": 2170.28,
+      "duration": 0.0,
+      "text": "to skip over all of these things uh but I<00:36:10.440><c> just</c><00:36:10.560><c> to</c><00:36:10.720><c> give</c><00:36:10.839><c> you</c><00:36:10.960><c> a</c><00:36:11.160><c> sense</c><00:36:11.400><c> of</c><00:36:11.599><c> how</c><00:36:11.800><c> hard</c>"
+    },
+    {
+      "start": 2172.03,
+      "duration": 0.0,
+      "text": "I just to give you a sense of how hard"
+    },
+    {
+      "start": 2172.04,
+      "duration": 0.0,
+      "text": "I just to give you a sense of how hard it<00:36:12.160><c> is</c><00:36:12.760><c> when</c><00:36:12.920><c> people</c><00:36:13.240><c> just</c><00:36:13.359><c> say</c><00:36:13.560><c> oh</c><00:36:13.680><c> I'm</c><00:36:13.760><c> going</c>"
+    },
+    {
+      "start": 2173.87,
+      "duration": 0.0,
+      "text": "it is when people just say oh I'm going"
+    },
+    {
+      "start": 2173.88,
+      "duration": 0.0,
+      "text": "it is when people just say oh I'm going to<00:36:13.960><c> train</c><00:36:14.160><c> on</c><00:36:14.440><c> internet</c><00:36:15.440><c> that's</c><00:36:15.640><c> a</c><00:36:15.760><c> lot</c><00:36:15.880><c> of</c>"
+    },
+    {
+      "start": 2176.19,
+      "duration": 0.0,
+      "text": "to train on internet that's a lot of"
+    },
+    {
+      "start": 2176.2,
+      "duration": 0.0,
+      "text": "to train on internet that's a lot of work<00:36:17.200><c> um</c><00:36:17.599><c> and</c><00:36:17.800><c> really</c><00:36:18.000><c> we</c><00:36:18.119><c> haven't</c><00:36:18.359><c> figured</c><00:36:18.599><c> it</c>"
+    },
+    {
+      "start": 2178.71,
+      "duration": 0.0,
+      "text": "work um and really we haven't figured it"
+    },
+    {
+      "start": 2178.72,
+      "duration": 0.0,
+      "text": "work um and really we haven't figured it out<00:36:19.160><c> yet</c><00:36:20.160><c> so</c><00:36:20.920><c> collecting</c><00:36:21.319><c> World</c><00:36:21.680><c> data</c><00:36:22.119><c> is</c><00:36:22.440><c> a</c>"
+    },
+    {
+      "start": 2182.67,
+      "duration": 0.0,
+      "text": "out yet so collecting World data is a"
+    },
+    {
+      "start": 2182.68,
+      "duration": 0.0,
+      "text": "out yet so collecting World data is a huge<00:36:23.079><c> part</c><00:36:23.319><c> of</c><00:36:23.520><c> practical</c><00:36:23.920><c> large</c><00:36:24.200><c> language</c>"
+    },
+    {
+      "start": 2184.51,
+      "duration": 0.0,
+      "text": "huge part of practical large language"
+    },
+    {
+      "start": 2184.52,
+      "duration": 0.0,
+      "text": "huge part of practical large language model<00:36:25.200><c> uh</c><00:36:25.319><c> some</c><00:36:25.520><c> might</c><00:36:25.640><c> say</c><00:36:25.880><c> it's</c><00:36:26.040><c> actually</c>"
+    },
+    {
+      "start": 2186.23,
+      "duration": 0.0,
+      "text": "model uh some might say it's actually"
+    },
+    {
+      "start": 2186.24,
+      "duration": 0.0,
+      "text": "model uh some might say it's actually the<00:36:26.480><c> key</c><00:36:26.920><c> yes</c>"
+    },
+    {
+      "start": 2188.27,
+      "duration": 0.0,
+      "text": "the key yes"
+    },
+    {
+      "start": 2188.28,
+      "duration": 0.0,
+      "text": "the key yes about<00:36:28.680><c> data</c><00:36:29.079><c> so</c><00:36:29.520><c> basic</c><00:36:29.880><c> question</c><00:36:30.119><c> so</c><00:36:30.319><c> usually</c>"
+    },
+    {
+      "start": 2190.67,
+      "duration": 0.0,
+      "text": "about data so basic question so usually"
+    },
+    {
+      "start": 2190.68,
+      "duration": 0.0,
+      "text": "about data so basic question so usually when<00:36:30.760><c> you</c><00:36:30.920><c> start</c><00:36:31.160><c> with</c><00:36:31.359><c> like</c><00:36:31.720><c> the</c><00:36:32.040><c> terabyte</c><00:36:33.040><c> of</c>"
+    },
+    {
+      "start": 2193.23,
+      "duration": 0.0,
+      "text": "when you start with like the terabyte of"
+    },
+    {
+      "start": 2193.24,
+      "duration": 0.0,
+      "text": "when you start with like the terabyte of data<00:36:33.800><c> after</c><00:36:34.000><c> I</c><00:36:34.160><c> go</c><00:36:34.280><c> through</c><00:36:34.480><c> all</c><00:36:34.640><c> that</c><00:36:34.839><c> steps</c>"
+    },
+    {
+      "start": 2195.27,
+      "duration": 0.0,
+      "text": "data after I go through all that steps"
+    },
+    {
+      "start": 2195.28,
+      "duration": 0.0,
+      "text": "data after I go through all that steps the<00:36:35.440><c> typical</c><00:36:35.839><c> amount</c><00:36:36.079><c> of</c><00:36:36.280><c> data</c><00:36:36.520><c> you</c><00:36:36.640><c> have</c><00:36:37.079><c> in</c>"
+    },
+    {
+      "start": 2197.91,
+      "duration": 0.0,
+      "text": "the typical amount of data you have in"
+    },
+    {
+      "start": 2197.92,
+      "duration": 0.0,
+      "text": "the typical amount of data you have in and<00:36:38.119><c> then</c><00:36:38.400><c> like</c><00:36:39.119><c> how</c><00:36:39.480><c> how</c><00:36:39.680><c> large</c><00:36:39.880><c> a</c><00:36:40.000><c> team</c><00:36:40.240><c> does</c>"
+    },
+    {
+      "start": 2200.349,
+      "duration": 0.0,
+      "text": "and then like how how large a team does"
+    },
+    {
+      "start": 2200.359,
+      "duration": 0.0,
+      "text": "and then like how how large a team does it<00:36:40.520><c> typically</c><00:36:40.960><c> think</c><00:36:41.119><c> to</c><00:36:41.319><c> go</c><00:36:41.520><c> through</c><00:36:41.800><c> all</c><00:36:41.960><c> the</c>"
+    },
+    {
+      "start": 2202.43,
+      "duration": 0.0,
+      "text": "it typically think to go through all the"
+    },
+    {
+      "start": 2202.44,
+      "duration": 0.0,
+      "text": "it typically think to go through all the steps<00:36:42.640><c> you</c><00:36:42.760><c> talk</c><00:36:43.040><c> about</c><00:36:43.599><c> so</c><00:36:43.880><c> how</c><00:36:44.280><c> is</c><00:36:44.359><c> the</c>"
+    },
+    {
+      "start": 2204.55,
+      "duration": 0.0,
+      "text": "steps you talk about so how is the"
+    },
+    {
+      "start": 2204.56,
+      "duration": 0.0,
+      "text": "steps you talk about so how is the question<00:36:44.800><c> how</c><00:36:45.000><c> large</c><00:36:45.200><c> is</c><00:36:45.319><c> the</c><00:36:45.520><c> data</c><00:36:45.880><c> after</c><00:36:46.119><c> you</c>"
+    },
+    {
+      "start": 2206.309,
+      "duration": 0.0,
+      "text": "question how large is the data after you"
+    },
+    {
+      "start": 2206.319,
+      "duration": 0.0,
+      "text": "question how large is the data after you filter<00:36:47.040><c> yeah</c><00:36:47.200><c> after</c><00:36:47.400><c> you</c><00:36:47.560><c> filter</c><00:36:47.960><c> and</c><00:36:48.079><c> then</c><00:36:48.200><c> to</c>"
+    },
+    {
+      "start": 2208.39,
+      "duration": 0.0,
+      "text": "filter yeah after you filter and then to"
+    },
+    {
+      "start": 2208.4,
+      "duration": 0.0,
+      "text": "filter yeah after you filter and then to go<00:36:48.520><c> through</c><00:36:48.680><c> all</c><00:36:48.839><c> the</c><00:36:49.000><c> step</c><00:36:49.240><c> how</c><00:36:49.400><c> large</c><00:36:49.640><c> a</c><00:36:49.800><c> team</c>"
+    },
+    {
+      "start": 2210.109,
+      "duration": 0.0,
+      "text": "go through all the step how large a team"
+    },
+    {
+      "start": 2210.119,
+      "duration": 0.0,
+      "text": "go through all the step how large a team do<00:36:50.200><c> you</c><00:36:50.359><c> need</c><00:36:50.640><c> to</c><00:36:50.839><c> go</c><00:36:51.000><c> through</c><00:36:51.280><c> like</c><00:36:51.560><c> the</c><00:36:52.240><c> the</c>"
+    },
+    {
+      "start": 2212.309,
+      "duration": 0.0,
+      "text": "do you need to go through like the the"
+    },
+    {
+      "start": 2212.319,
+      "duration": 0.0,
+      "text": "do you need to go through like the the other<00:36:52.599><c> fation</c><00:36:53.520><c> sttion</c><00:36:54.520><c> uh</c><00:36:54.960><c> how</c><00:36:55.160><c> slow</c><00:36:55.520><c> is</c><00:36:55.640><c> it</c><00:36:55.960><c> or</c>"
+    },
+    {
+      "start": 2216.63,
+      "duration": 0.0,
+      "text": "other fation sttion uh how slow is it or"
+    },
+    {
+      "start": 2216.64,
+      "duration": 0.0,
+      "text": "other fation sttion uh how slow is it or how<00:36:56.920><c> like</c><00:36:57.200><c> how</c><00:36:57.839><c> how</c><00:36:58.000><c> many</c><00:36:58.280><c> people</c><00:36:58.520><c> would</c><00:36:58.680><c> you</c>"
+    },
+    {
+      "start": 2218.87,
+      "duration": 0.0,
+      "text": "how like how how many people would you"
+    },
+    {
+      "start": 2218.88,
+      "duration": 0.0,
+      "text": "how like how how many people would you need<00:36:59.880><c> to</c><00:37:00.079><c> be</c><00:37:00.280><c> able</c><00:37:00.480><c> to</c><00:37:00.680><c> do</c><00:37:01.200><c> this</c><00:37:02.200><c> uh</c><00:37:02.520><c> okay</c>"
+    },
+    {
+      "start": 2222.67,
+      "duration": 0.0,
+      "text": "need to be able to do this uh okay"
+    },
+    {
+      "start": 2222.68,
+      "duration": 0.0,
+      "text": "need to be able to do this uh okay that's<00:37:02.800><c> a</c><00:37:02.920><c> great</c><00:37:03.160><c> question</c><00:37:03.520><c> I'm</c><00:37:03.599><c> going</c><00:37:03.720><c> to</c>"
+    },
+    {
+      "start": 2224.349,
+      "duration": 0.0,
+      "text": "that's a great question I'm going to"
+    },
+    {
+      "start": 2224.359,
+      "duration": 0.0,
+      "text": "that's a great question I'm going to somewhat<00:37:04.839><c> answer</c><00:37:05.240><c> about</c><00:37:05.560><c> the</c><00:37:05.720><c> data</c><00:37:06.520><c> uh</c><00:37:06.800><c> how</c>"
+    },
+    {
+      "start": 2226.95,
+      "duration": 0.0,
+      "text": "somewhat answer about the data uh how"
+    },
+    {
+      "start": 2226.96,
+      "duration": 0.0,
+      "text": "somewhat answer about the data uh how large<00:37:07.200><c> is</c><00:37:07.280><c> the</c><00:37:07.440><c> data</c><00:37:07.720><c> set</c><00:37:08.280><c> uh</c><00:37:08.440><c> at</c><00:37:08.520><c> the</c><00:37:08.599><c> end</c><00:37:08.760><c> of</c>"
+    },
+    {
+      "start": 2228.87,
+      "duration": 0.0,
+      "text": "large is the data set uh at the end of"
+    },
+    {
+      "start": 2228.88,
+      "duration": 0.0,
+      "text": "large is the data set uh at the end of this<00:37:09.079><c> slide</c><00:37:10.079><c> uh</c><00:37:10.560><c> for</c><00:37:11.560><c> number</c><00:37:11.800><c> of</c><00:37:12.000><c> people</c><00:37:12.240><c> that</c>"
+    },
+    {
+      "start": 2232.39,
+      "duration": 0.0,
+      "text": "this slide uh for number of people that"
+    },
+    {
+      "start": 2232.4,
+      "duration": 0.0,
+      "text": "this slide uh for number of people that work<00:37:12.640><c> on</c>"
+    },
+    {
+      "start": 2233.51,
+      "duration": 0.0,
+      "text": "work on"
+    },
+    {
+      "start": 2233.52,
+      "duration": 0.0,
+      "text": "work on it<00:37:14.520><c> um</c><00:37:14.760><c> that's</c><00:37:14.920><c> a</c><00:37:15.079><c> good</c><00:37:15.280><c> question</c><00:37:15.560><c> I'm</c>"
+    },
+    {
+      "start": 2235.75,
+      "duration": 0.0,
+      "text": "it um that's a good question I'm"
+    },
+    {
+      "start": 2235.76,
+      "duration": 0.0,
+      "text": "it um that's a good question I'm actually<00:37:16.400><c> not</c><00:37:16.680><c> quite</c><00:37:16.880><c> sure</c><00:37:17.240><c> but</c><00:37:17.359><c> I</c><00:37:17.480><c> would</c>"
+    },
+    {
+      "start": 2238.51,
+      "duration": 0.0,
+      "text": "actually not quite sure but I would"
+    },
+    {
+      "start": 2238.52,
+      "duration": 0.0,
+      "text": "actually not quite sure but I would say<00:37:19.520><c> yeah</c><00:37:19.680><c> I</c><00:37:19.800><c> actually</c><00:37:20.040><c> don't</c><00:37:20.920><c> quite</c><00:37:21.920><c> no</c><00:37:22.319><c> but</c><00:37:22.480><c> I</c>"
+    },
+    {
+      "start": 2242.55,
+      "duration": 0.0,
+      "text": "say yeah I actually don't quite no but I"
+    },
+    {
+      "start": 2242.56,
+      "duration": 0.0,
+      "text": "say yeah I actually don't quite no but I would<00:37:22.720><c> say</c><00:37:23.119><c> it's</c><00:37:23.480><c> probably</c><00:37:23.720><c> even</c><00:37:23.960><c> bigger</c><00:37:24.240><c> than</c>"
+    },
+    {
+      "start": 2244.349,
+      "duration": 0.0,
+      "text": "would say it's probably even bigger than"
+    },
+    {
+      "start": 2244.359,
+      "duration": 0.0,
+      "text": "would say it's probably even bigger than the<00:37:24.480><c> number</c><00:37:24.680><c> of</c><00:37:24.839><c> people</c><00:37:25.119><c> that</c><00:37:25.280><c> work</c><00:37:25.800><c> on</c><00:37:26.040><c> kind</c>"
+    },
+    {
+      "start": 2246.19,
+      "duration": 0.0,
+      "text": "the number of people that work on kind"
+    },
+    {
+      "start": 2246.2,
+      "duration": 0.0,
+      "text": "the number of people that work on kind of<00:37:26.440><c> the</c><00:37:27.200><c> two</c><00:37:27.520><c> tuning</c><00:37:27.920><c> of</c><00:37:28.040><c> the</c><00:37:28.200><c> pre-training</c><00:37:28.720><c> of</c>"
+    },
+    {
+      "start": 2248.829,
+      "duration": 0.0,
+      "text": "of the two tuning of the pre-training of"
+    },
+    {
+      "start": 2248.839,
+      "duration": 0.0,
+      "text": "of the two tuning of the pre-training of the<00:37:29.000><c> model</c><00:37:29.839><c> uh</c><00:37:29.920><c> so</c><00:37:30.079><c> the</c><00:37:30.280><c> data</c><00:37:30.560><c> is</c><00:37:30.760><c> bigger</c><00:37:31.440><c> than</c>"
+    },
+    {
+      "start": 2251.63,
+      "duration": 0.0,
+      "text": "the model uh so the data is bigger than"
+    },
+    {
+      "start": 2251.64,
+      "duration": 0.0,
+      "text": "the model uh so the data is bigger than kind<00:37:31.760><c> of</c><00:37:31.880><c> the</c><00:37:32.040><c> modeling</c><00:37:32.640><c> aspect</c><00:37:34.079><c> um</c><00:37:35.079><c> yeah</c><00:37:35.359><c> I</c><00:37:35.640><c> I</c>"
+    },
+    {
+      "start": 2255.79,
+      "duration": 0.0,
+      "text": "kind of the modeling aspect um yeah I I"
+    },
+    {
+      "start": 2255.8,
+      "duration": 0.0,
+      "text": "kind of the modeling aspect um yeah I I don't<00:37:36.000><c> think</c><00:37:36.160><c> I</c><00:37:36.319><c> have</c><00:37:36.440><c> a</c><00:37:36.680><c> good</c><00:37:37.560><c> sense</c><00:37:37.880><c> I</c><00:37:37.960><c> would</c>"
+    },
+    {
+      "start": 2258.069,
+      "duration": 0.0,
+      "text": "don't think I have a good sense I would"
+    },
+    {
+      "start": 2258.079,
+      "duration": 0.0,
+      "text": "don't think I have a good sense I would say<00:37:38.240><c> probably</c><00:37:38.440><c> in</c><00:37:38.560><c> Lama's</c><00:37:39.040><c> team</c><00:37:39.720><c> which</c><00:37:40.000><c> have</c>"
+    },
+    {
+      "start": 2260.23,
+      "duration": 0.0,
+      "text": "say probably in Lama's team which have"
+    },
+    {
+      "start": 2260.24,
+      "duration": 0.0,
+      "text": "say probably in Lama's team which have like<00:37:40.400><c> 70</c><00:37:40.839><c> years</c><00:37:41.160><c> people</c><00:37:41.400><c> I</c><00:37:41.440><c> would</c><00:37:41.560><c> say</c><00:37:41.720><c> maybe</c>"
+    },
+    {
+      "start": 2262.19,
+      "duration": 0.0,
+      "text": "like 70 years people I would say maybe"
+    },
+    {
+      "start": 2262.2,
+      "duration": 0.0,
+      "text": "like 70 years people I would say maybe 15<00:37:42.680><c> work</c><00:37:42.920><c> on</c><00:37:43.240><c> data</c><00:37:44.240><c> uh</c><00:37:45.079><c> I</c><00:37:45.760><c> yeah</c><00:37:46.760><c> all</c><00:37:46.960><c> these</c>"
+    },
+    {
+      "start": 2267.069,
+      "duration": 0.0,
+      "text": "15 work on data uh I yeah all these"
+    },
+    {
+      "start": 2267.079,
+      "duration": 0.0,
+      "text": "15 work on data uh I yeah all these things<00:37:47.240><c> you</c><00:37:47.359><c> don't</c><00:37:47.520><c> need</c><00:37:47.680><c> that</c><00:37:47.839><c> many</c><00:37:48.000><c> people</c>"
+    },
+    {
+      "start": 2268.19,
+      "duration": 0.0,
+      "text": "things you don't need that many people"
+    },
+    {
+      "start": 2268.2,
+      "duration": 0.0,
+      "text": "things you don't need that many people you<00:37:48.319><c> need</c><00:37:48.440><c> a</c><00:37:48.520><c> lot</c><00:37:48.680><c> of</c><00:37:48.760><c> computer</c><00:37:49.200><c> so</c><00:37:49.560><c> because</c>"
+    },
+    {
+      "start": 2269.829,
+      "duration": 0.0,
+      "text": "you need a lot of computer so because"
+    },
+    {
+      "start": 2269.839,
+      "duration": 0.0,
+      "text": "you need a lot of computer so because for<00:37:50.040><c> data</c><00:37:50.240><c> you</c><00:37:50.319><c> need</c><00:37:50.440><c> a</c><00:37:50.520><c> lot</c><00:37:50.599><c> of</c><00:37:51.079><c> CPUs</c><00:37:52.079><c> um</c><00:37:53.000><c> so</c>"
+    },
+    {
+      "start": 2273.19,
+      "duration": 0.0,
+      "text": "for data you need a lot of CPUs um so"
+    },
+    {
+      "start": 2273.2,
+      "duration": 0.0,
+      "text": "for data you need a lot of CPUs um so yeah<00:37:53.319><c> and</c><00:37:53.480><c> I'll</c><00:37:53.640><c> answer</c><00:37:53.880><c> the</c><00:37:54.119><c> second</c><00:37:54.400><c> question</c>"
+    },
+    {
+      "start": 2274.91,
+      "duration": 0.0,
+      "text": "yeah and I'll answer the second question"
+    },
+    {
+      "start": 2274.92,
+      "duration": 0.0,
+      "text": "yeah and I'll answer the second question at<00:37:55.000><c> the</c><00:37:55.119><c> end</c><00:37:55.240><c> of</c><00:37:55.359><c> this</c><00:37:55.680><c> slide</c><00:37:56.680><c> so</c><00:37:57.040><c> as</c><00:37:57.160><c> I</c><00:37:57.280><c> just</c>"
+    },
+    {
+      "start": 2277.79,
+      "duration": 0.0,
+      "text": "at the end of this slide so as I just"
+    },
+    {
+      "start": 2277.8,
+      "duration": 0.0,
+      "text": "at the end of this slide so as I just kind<00:37:57.920><c> of</c><00:37:58.480><c> alluded</c><00:37:58.920><c> to</c><00:37:59.599><c> really</c><00:37:59.839><c> we</c><00:37:59.960><c> haven't</c>"
+    },
+    {
+      "start": 2280.19,
+      "duration": 0.0,
+      "text": "kind of alluded to really we haven't"
+    },
+    {
+      "start": 2280.2,
+      "duration": 0.0,
+      "text": "kind of alluded to really we haven't solved<00:38:00.599><c> data</c><00:38:01.000><c> at</c><00:38:01.160><c> all</c><00:38:01.359><c> for</c><00:38:01.599><c> pre-training</c><00:38:02.240><c> so</c>"
+    },
+    {
+      "start": 2282.349,
+      "duration": 0.0,
+      "text": "solved data at all for pre-training so"
+    },
+    {
+      "start": 2282.359,
+      "duration": 0.0,
+      "text": "solved data at all for pre-training so there's<00:38:02.520><c> a</c><00:38:02.640><c> lot</c><00:38:02.760><c> of</c><00:38:02.880><c> research</c><00:38:03.280><c> that</c><00:38:03.480><c> that</c><00:38:03.599><c> has</c>"
+    },
+    {
+      "start": 2283.71,
+      "duration": 0.0,
+      "text": "there's a lot of research that that has"
+    },
+    {
+      "start": 2283.72,
+      "duration": 0.0,
+      "text": "there's a lot of research that that has to<00:38:03.839><c> be</c><00:38:04.000><c> done</c><00:38:04.480><c> first</c><00:38:04.760><c> how</c><00:38:04.839><c> do</c><00:38:04.960><c> you</c><00:38:05.160><c> process</c>"
+    },
+    {
+      "start": 2285.589,
+      "duration": 0.0,
+      "text": "to be done first how do you process"
+    },
+    {
+      "start": 2285.599,
+      "duration": 0.0,
+      "text": "to be done first how do you process these<00:38:05.720><c> things</c><00:38:05.920><c> super</c><00:38:06.200><c> efficiently</c><00:38:07.200><c> uh</c><00:38:07.319><c> second</c>"
+    },
+    {
+      "start": 2287.63,
+      "duration": 0.0,
+      "text": "these things super efficiently uh second"
+    },
+    {
+      "start": 2287.64,
+      "duration": 0.0,
+      "text": "these things super efficiently uh second how<00:38:07.760><c> do</c><00:38:07.880><c> you</c><00:38:08.000><c> balance</c><00:38:08.440><c> kind</c><00:38:08.560><c> of</c><00:38:08.760><c> like</c><00:38:09.200><c> all</c><00:38:09.319><c> of</c>"
+    },
+    {
+      "start": 2289.47,
+      "duration": 0.0,
+      "text": "how do you balance kind of like all of"
+    },
+    {
+      "start": 2289.48,
+      "duration": 0.0,
+      "text": "how do you balance kind of like all of these<00:38:09.640><c> different</c><00:38:09.880><c> domains</c><00:38:10.720><c> uh</c><00:38:10.839><c> can</c><00:38:10.960><c> you</c><00:38:11.079><c> do</c>"
+    },
+    {
+      "start": 2291.27,
+      "duration": 0.0,
+      "text": "these different domains uh can you do"
+    },
+    {
+      "start": 2291.28,
+      "duration": 0.0,
+      "text": "these different domains uh can you do synthetic<00:38:11.760><c> data</c><00:38:12.000><c> generation</c><00:38:12.480><c> that's</c>"
+    },
+    {
+      "start": 2292.67,
+      "duration": 0.0,
+      "text": "synthetic data generation that's"
+    },
+    {
+      "start": 2292.68,
+      "duration": 0.0,
+      "text": "synthetic data generation that's actually<00:38:12.839><c> a</c><00:38:13.000><c> big</c><00:38:13.119><c> one</c><00:38:13.319><c> right</c><00:38:13.520><c> now</c><00:38:14.319><c> uh</c><00:38:14.560><c> and</c>"
+    },
+    {
+      "start": 2295.069,
+      "duration": 0.0,
+      "text": "actually a big one right now uh and"
+    },
+    {
+      "start": 2295.079,
+      "duration": 0.0,
+      "text": "actually a big one right now uh and because<00:38:15.319><c> we</c><00:38:15.440><c> don't</c><00:38:15.760><c> have</c><00:38:16.480><c> uh</c><00:38:16.599><c> we'll</c><00:38:16.800><c> talk</c>"
+    },
+    {
+      "start": 2296.95,
+      "duration": 0.0,
+      "text": "because we don't have uh we'll talk"
+    },
+    {
+      "start": 2296.96,
+      "duration": 0.0,
+      "text": "because we don't have uh we'll talk about<00:38:17.160><c> that</c><00:38:17.280><c> later</c><00:38:17.599><c> we</c><00:38:17.720><c> don't</c><00:38:17.920><c> have</c><00:38:18.119><c> enough</c>"
+    },
+    {
+      "start": 2298.39,
+      "duration": 0.0,
+      "text": "about that later we don't have enough"
+    },
+    {
+      "start": 2298.4,
+      "duration": 0.0,
+      "text": "about that later we don't have enough data<00:38:18.960><c> on</c><00:38:19.119><c> the</c><00:38:19.359><c> internet</c><00:38:20.359><c> um</c><00:38:20.800><c> can</c><00:38:20.960><c> you</c><00:38:21.079><c> use</c>"
+    },
+    {
+      "start": 2301.309,
+      "duration": 0.0,
+      "text": "data on the internet um can you use"
+    },
+    {
+      "start": 2301.319,
+      "duration": 0.0,
+      "text": "data on the internet um can you use multimodal<00:38:22.079><c> data</c><00:38:22.440><c> instead</c><00:38:22.680><c> of</c><00:38:22.839><c> just</c><00:38:23.000><c> text</c>"
+    },
+    {
+      "start": 2303.349,
+      "duration": 0.0,
+      "text": "multimodal data instead of just text"
+    },
+    {
+      "start": 2303.359,
+      "duration": 0.0,
+      "text": "multimodal data instead of just text data<00:38:23.880><c> and</c><00:38:24.040><c> how</c><00:38:24.200><c> does</c><00:38:24.400><c> that</c><00:38:24.599><c> improve</c><00:38:25.200><c> even</c><00:38:25.440><c> your</c>"
+    },
+    {
+      "start": 2305.67,
+      "duration": 0.0,
+      "text": "data and how does that improve even your"
+    },
+    {
+      "start": 2305.68,
+      "duration": 0.0,
+      "text": "data and how does that improve even your text<00:38:26.319><c> performance</c><00:38:27.319><c> um</c>"
+    },
+    {
+      "start": 2308.349,
+      "duration": 0.0,
+      "text": "text performance um"
+    },
+    {
+      "start": 2308.359,
+      "duration": 0.0,
+      "text": "text performance um there's<00:38:28.520><c> a</c><00:38:28.640><c> lot</c><00:38:28.760><c> of</c><00:38:28.880><c> seccy</c><00:38:29.560><c> because</c><00:38:29.760><c> really</c>"
+    },
+    {
+      "start": 2309.99,
+      "duration": 0.0,
+      "text": "there's a lot of seccy because really"
+    },
+    {
+      "start": 2310.0,
+      "duration": 0.0,
+      "text": "there's a lot of seccy because really this<00:38:30.119><c> is</c><00:38:30.319><c> the</c><00:38:30.520><c> key</c><00:38:31.200><c> of</c><00:38:31.480><c> most</c><00:38:31.680><c> of</c><00:38:31.800><c> the</c><00:38:31.960><c> pre-train</c>"
+    },
+    {
+      "start": 2312.51,
+      "duration": 0.0,
+      "text": "this is the key of most of the pre-train"
+    },
+    {
+      "start": 2312.52,
+      "duration": 0.0,
+      "text": "this is the key of most of the pre-train pre-trained<00:38:32.920><c> large</c><00:38:33.160><c> language</c><00:38:33.440><c> models</c><00:38:34.280><c> so</c><00:38:34.440><c> for</c>"
+    },
+    {
+      "start": 2314.63,
+      "duration": 0.0,
+      "text": "pre-trained large language models so for"
+    },
+    {
+      "start": 2314.64,
+      "duration": 0.0,
+      "text": "pre-trained large language models so for competitive<00:38:35.119><c> Dynamics</c><00:38:36.119><c> uh</c><00:38:36.280><c> usually</c><00:38:36.720><c> these</c>"
+    },
+    {
+      "start": 2317.15,
+      "duration": 0.0,
+      "text": "competitive Dynamics uh usually these"
+    },
+    {
+      "start": 2317.16,
+      "duration": 0.0,
+      "text": "competitive Dynamics uh usually these these<00:38:37.640><c> um</c><00:38:38.480><c> these</c><00:38:39.119><c> companies</c><00:38:39.560><c> don't</c><00:38:39.839><c> talk</c>"
+    },
+    {
+      "start": 2320.03,
+      "duration": 0.0,
+      "text": "these um these companies don't talk"
+    },
+    {
+      "start": 2320.04,
+      "duration": 0.0,
+      "text": "these um these companies don't talk about<00:38:40.280><c> how</c><00:38:40.400><c> they</c><00:38:40.520><c> do</c><00:38:40.680><c> the</c><00:38:40.800><c> data</c><00:38:41.079><c> collection</c>"
+    },
+    {
+      "start": 2321.91,
+      "duration": 0.0,
+      "text": "about how they do the data collection"
+    },
+    {
+      "start": 2321.92,
+      "duration": 0.0,
+      "text": "about how they do the data collection and<00:38:42.040><c> also</c><00:38:42.240><c> there's</c><00:38:42.400><c> a</c><00:38:42.520><c> copyright</c><00:38:42.960><c> liability</c>"
+    },
+    {
+      "start": 2323.55,
+      "duration": 0.0,
+      "text": "and also there's a copyright liability"
+    },
+    {
+      "start": 2323.56,
+      "duration": 0.0,
+      "text": "and also there's a copyright liability issue<00:38:44.040><c> they</c><00:38:44.200><c> definitely</c><00:38:44.480><c> don't</c><00:38:44.560><c> want</c><00:38:44.680><c> to</c><00:38:44.800><c> tell</c>"
+    },
+    {
+      "start": 2324.95,
+      "duration": 0.0,
+      "text": "issue they definitely don't want to tell"
+    },
+    {
+      "start": 2324.96,
+      "duration": 0.0,
+      "text": "issue they definitely don't want to tell you<00:38:45.079><c> that</c><00:38:45.200><c> they've</c><00:38:45.359><c> trained</c><00:38:45.640><c> on</c><00:38:45.800><c> books</c><00:38:46.079><c> even</c>"
+    },
+    {
+      "start": 2326.23,
+      "duration": 0.0,
+      "text": "you that they've trained on books even"
+    },
+    {
+      "start": 2326.24,
+      "duration": 0.0,
+      "text": "you that they've trained on books even though<00:38:46.400><c> they</c><00:38:46.560><c> did</c><00:38:47.240><c> um</c><00:38:47.440><c> because</c><00:38:47.640><c> if</c><00:38:47.760><c> not</c><00:38:47.920><c> you</c>"
+    },
+    {
+      "start": 2328.03,
+      "duration": 0.0,
+      "text": "though they did um because if not you"
+    },
+    {
+      "start": 2328.04,
+      "duration": 0.0,
+      "text": "though they did um because if not you can<00:38:48.880><c> uh</c><00:38:48.960><c> sue</c><00:38:49.480><c> them</c><00:38:50.480><c> uh</c><00:38:50.640><c> common</c><00:38:51.000><c> academic</c>"
+    },
+    {
+      "start": 2331.39,
+      "duration": 0.0,
+      "text": "can uh sue them uh common academic"
+    },
+    {
+      "start": 2331.4,
+      "duration": 0.0,
+      "text": "can uh sue them uh common academic benchmarks<00:38:52.200><c> uh</c><00:38:52.319><c> so</c><00:38:52.480><c> that</c><00:38:52.599><c> will</c><00:38:52.800><c> kind</c><00:38:52.920><c> of</c>"
+    },
+    {
+      "start": 2333.069,
+      "duration": 0.0,
+      "text": "benchmarks uh so that will kind of"
+    },
+    {
+      "start": 2333.079,
+      "duration": 0.0,
+      "text": "benchmarks uh so that will kind of answer<00:38:53.359><c> what</c><00:38:53.480><c> you</c><00:38:53.680><c> asked</c><00:38:54.599><c> um</c><00:38:54.880><c> it</c><00:38:55.079><c> started</c><00:38:55.760><c> so</c>"
+    },
+    {
+      "start": 2335.95,
+      "duration": 0.0,
+      "text": "answer what you asked um it started so"
+    },
+    {
+      "start": 2335.96,
+      "duration": 0.0,
+      "text": "answer what you asked um it started so those<00:38:56.119><c> are</c><00:38:56.240><c> the</c><00:38:56.359><c> smaller</c><00:38:56.720><c> ones</c><00:38:57.520><c> it's</c><00:38:57.760><c> the</c>"
+    },
+    {
+      "start": 2337.87,
+      "duration": 0.0,
+      "text": "those are the smaller ones it's the"
+    },
+    {
+      "start": 2337.88,
+      "duration": 0.0,
+      "text": "those are the smaller ones it's the names<00:38:58.079><c> are</c><00:38:58.200><c> not</c><00:38:58.400><c> that</c><00:38:58.520><c> important</c><00:38:58.960><c> but</c><00:38:59.079><c> it</c>"
+    },
+    {
+      "start": 2339.19,
+      "duration": 0.0,
+      "text": "names are not that important but it"
+    },
+    {
+      "start": 2339.2,
+      "duration": 0.0,
+      "text": "names are not that important but it started<00:38:59.520><c> from</c><00:39:00.040><c> around</c><00:39:00.359><c> 150</c><00:39:01.079><c> billion</c><00:39:01.440><c> tokens</c>"
+    },
+    {
+      "start": 2342.069,
+      "duration": 0.0,
+      "text": "started from around 150 billion tokens"
+    },
+    {
+      "start": 2342.079,
+      "duration": 0.0,
+      "text": "started from around 150 billion tokens which<00:39:02.359><c> around</c><00:39:02.800><c> uh</c><00:39:02.920><c> 800</c><00:39:03.480><c> GB</c><00:39:03.880><c> of</c><00:39:04.040><c> data</c><00:39:04.720><c> now</c><00:39:04.920><c> it's</c>"
+    },
+    {
+      "start": 2345.069,
+      "duration": 0.0,
+      "text": "which around uh 800 GB of data now it's"
+    },
+    {
+      "start": 2345.079,
+      "duration": 0.0,
+      "text": "which around uh 800 GB of data now it's around<00:39:05.400><c> 15</c><00:39:05.760><c> trillion</c><00:39:06.160><c> of</c><00:39:06.359><c> to</c><00:39:06.640><c> 15</c><00:39:07.040><c> trillion</c>"
+    },
+    {
+      "start": 2347.43,
+      "duration": 0.0,
+      "text": "around 15 trillion of to 15 trillion"
+    },
+    {
+      "start": 2347.44,
+      "duration": 0.0,
+      "text": "around 15 trillion of to 15 trillion tokens<00:39:08.119><c> which</c><00:39:08.280><c> is</c><00:39:08.480><c> also</c><00:39:09.440><c> uh</c><00:39:09.599><c> the</c><00:39:09.839><c> size</c><00:39:10.200><c> of</c><00:39:10.599><c> the</c>"
+    },
+    {
+      "start": 2350.75,
+      "duration": 0.0,
+      "text": "tokens which is also uh the size of the"
+    },
+    {
+      "start": 2350.76,
+      "duration": 0.0,
+      "text": "tokens which is also uh the size of the models<00:39:11.200><c> that</c><00:39:11.359><c> are</c><00:39:11.880><c> right</c><00:39:12.040><c> now</c><00:39:12.200><c> the</c><00:39:12.319><c> best</c>"
+    },
+    {
+      "start": 2352.51,
+      "duration": 0.0,
+      "text": "models that are right now the best"
+    },
+    {
+      "start": 2352.52,
+      "duration": 0.0,
+      "text": "models that are right now the best models<00:39:12.760><c> are</c><00:39:12.920><c> probably</c><00:39:13.119><c> trained</c><00:39:13.400><c> on</c><00:39:13.560><c> that</c>"
+    },
+    {
+      "start": 2353.67,
+      "duration": 0.0,
+      "text": "models are probably trained on that"
+    },
+    {
+      "start": 2353.68,
+      "duration": 0.0,
+      "text": "models are probably trained on that amount<00:39:13.880><c> of</c><00:39:14.000><c> data</c><00:39:14.480><c> so</c><00:39:14.680><c> 15</c><00:39:15.000><c> trillion</c><00:39:15.400><c> tokens</c><00:39:16.400><c> uh</c>"
+    },
+    {
+      "start": 2356.589,
+      "duration": 0.0,
+      "text": "amount of data so 15 trillion tokens uh"
+    },
+    {
+      "start": 2356.599,
+      "duration": 0.0,
+      "text": "amount of data so 15 trillion tokens uh which<00:39:16.800><c> is</c><00:39:17.520><c> probably</c><00:39:18.520><c> I</c><00:39:18.680><c> guess</c><00:39:18.960><c> two</c><00:39:19.240><c> order</c><00:39:19.480><c> of</c>"
+    },
+    {
+      "start": 2359.589,
+      "duration": 0.0,
+      "text": "which is probably I guess two order of"
+    },
+    {
+      "start": 2359.599,
+      "duration": 0.0,
+      "text": "which is probably I guess two order of manage<00:39:19.960><c> bigger</c><00:39:20.200><c> than</c><00:39:20.359><c> that</c><00:39:20.520><c> so</c><00:39:21.280><c> 80</c><00:39:22.280><c> uh</c><00:39:22.440><c> E3</c><00:39:23.040><c> gab</c>"
+    },
+    {
+      "start": 2363.829,
+      "duration": 0.0,
+      "text": "manage bigger than that so 80 uh E3 gab"
+    },
+    {
+      "start": 2363.839,
+      "duration": 0.0,
+      "text": "manage bigger than that so 80 uh E3 gab so<00:39:24.119><c> that</c><00:39:24.240><c> would</c><00:39:24.440><c> be</c>"
+    },
+    {
+      "start": 2365.67,
+      "duration": 0.0,
+      "text": "so that would be"
+    },
+    {
+      "start": 2365.68,
+      "duration": 0.0,
+      "text": "so that would be around<00:39:26.680><c> 100</c><00:39:26.920><c> to</c><00:39:27.400><c> thousand</c><00:39:27.760><c> times</c><00:39:28.640><c> uh</c>"
+    },
+    {
+      "start": 2368.87,
+      "duration": 0.0,
+      "text": "around 100 to thousand times uh"
+    },
+    {
+      "start": 2368.88,
+      "duration": 0.0,
+      "text": "around 100 to thousand times uh filtering<00:39:29.359><c> of</c><00:39:29.480><c> the</c><00:39:29.599><c> common</c><00:39:29.960><c> crawl</c><00:39:30.640><c> if</c><00:39:30.760><c> I'm</c><00:39:30.920><c> not</c>"
+    },
+    {
+      "start": 2371.47,
+      "duration": 0.0,
+      "text": "filtering of the common crawl if I'm not"
+    },
+    {
+      "start": 2371.48,
+      "duration": 0.0,
+      "text": "filtering of the common crawl if I'm not mistaken<00:39:32.480><c> um</c><00:39:33.040><c> so</c><00:39:33.280><c> yeah</c><00:39:33.880><c> one</c><00:39:34.119><c> very</c><00:39:34.599><c> one</c><00:39:34.839><c> very</c><00:39:35.240><c> uh</c>"
+    },
+    {
+      "start": 2375.43,
+      "duration": 0.0,
+      "text": "mistaken um so yeah one very one very uh"
+    },
+    {
+      "start": 2375.44,
+      "duration": 0.0,
+      "text": "mistaken um so yeah one very one very uh famous<00:39:35.720><c> one</c><00:39:35.920><c> is</c><00:39:36.079><c> the</c><00:39:36.240><c> pile</c><00:39:37.240><c> so</c><00:39:37.440><c> this</c><00:39:37.560><c> is</c>"
+    },
+    {
+      "start": 2377.87,
+      "duration": 0.0,
+      "text": "famous one is the pile so this is"
+    },
+    {
+      "start": 2377.88,
+      "duration": 0.0,
+      "text": "famous one is the pile so this is academic<00:39:38.280><c> Benchmark</c><00:39:38.720><c> of</c><00:39:38.839><c> the</c><00:39:38.960><c> pile</c><00:39:39.440><c> and</c><00:39:39.560><c> we</c>"
+    },
+    {
+      "start": 2379.63,
+      "duration": 0.0,
+      "text": "academic Benchmark of the pile and we"
+    },
+    {
+      "start": 2379.64,
+      "duration": 0.0,
+      "text": "academic Benchmark of the pile and we can<00:39:39.880><c> just</c><00:39:40.079><c> look</c><00:39:40.280><c> at</c><00:39:40.760><c> what</c><00:39:40.920><c> distribution</c><00:39:41.400><c> of</c>"
+    },
+    {
+      "start": 2381.55,
+      "duration": 0.0,
+      "text": "can just look at what distribution of"
+    },
+    {
+      "start": 2381.56,
+      "duration": 0.0,
+      "text": "can just look at what distribution of data<00:39:41.839><c> they</c><00:39:42.040><c> have</c><00:39:42.640><c> it's</c><00:39:42.839><c> things</c><00:39:43.200><c> like</c><00:39:44.079><c> um</c>"
+    },
+    {
+      "start": 2384.43,
+      "duration": 0.0,
+      "text": "data they have it's things like um"
+    },
+    {
+      "start": 2384.44,
+      "duration": 0.0,
+      "text": "data they have it's things like um archive<00:39:45.319><c> PBM</c><00:39:45.920><c> Central</c><00:39:46.920><c> uh</c><00:39:47.079><c> which</c><00:39:47.200><c> is</c><00:39:47.319><c> all</c><00:39:47.520><c> the</c>"
+    },
+    {
+      "start": 2387.91,
+      "duration": 0.0,
+      "text": "archive PBM Central uh which is all the"
+    },
+    {
+      "start": 2387.92,
+      "duration": 0.0,
+      "text": "archive PBM Central uh which is all the the<00:39:48.280><c> biology</c><00:39:48.839><c> stuff</c><00:39:49.839><c> uh</c><00:39:50.640><c> here</c><00:39:51.119><c> it's</c><00:39:51.520><c> Wikipedia</c>"
+    },
+    {
+      "start": 2392.15,
+      "duration": 0.0,
+      "text": "the biology stuff uh here it's Wikipedia"
+    },
+    {
+      "start": 2392.16,
+      "duration": 0.0,
+      "text": "the biology stuff uh here it's Wikipedia you<00:39:52.280><c> see</c><00:39:52.599><c> stack</c><00:39:53.200><c> exchange</c><00:39:54.200><c> um</c><00:39:54.720><c> some</c><00:39:55.079><c> GitHub</c>"
+    },
+    {
+      "start": 2396.069,
+      "duration": 0.0,
+      "text": "you see stack exchange um some GitHub"
+    },
+    {
+      "start": 2396.079,
+      "duration": 0.0,
+      "text": "you see stack exchange um some GitHub and<00:39:56.280><c> some</c><00:39:56.520><c> books</c><00:39:56.880><c> and</c><00:39:57.000><c> things</c><00:39:57.200><c> like</c><00:39:57.599><c> this</c><00:39:58.280><c> um</c>"
+    },
+    {
+      "start": 2398.55,
+      "duration": 0.0,
+      "text": "and some books and things like this um"
+    },
+    {
+      "start": 2398.56,
+      "duration": 0.0,
+      "text": "and some books and things like this um again<00:39:58.760><c> this</c><00:39:58.839><c> is</c><00:39:58.960><c> on</c><00:39:59.079><c> the</c><00:39:59.200><c> smaller</c><00:39:59.599><c> side</c><00:39:59.960><c> so</c>"
+    },
+    {
+      "start": 2400.19,
+      "duration": 0.0,
+      "text": "again this is on the smaller side so"
+    },
+    {
+      "start": 2400.2,
+      "duration": 0.0,
+      "text": "again this is on the smaller side so this<00:40:00.359><c> is</c><00:40:00.720><c> if</c><00:40:00.800><c> we</c><00:40:00.960><c> look</c><00:40:01.160><c> at</c><00:40:01.319><c> here</c><00:40:01.480><c> this</c><00:40:01.560><c> is</c><00:40:01.640><c> on</c>"
+    },
+    {
+      "start": 2401.79,
+      "duration": 0.0,
+      "text": "this is if we look at here this is on"
+    },
+    {
+      "start": 2401.8,
+      "duration": 0.0,
+      "text": "this is if we look at here this is on 280b<00:40:02.760><c> so</c><00:40:02.920><c> in</c><00:40:03.079><c> reality</c><00:40:03.400><c> it's</c><00:40:03.560><c> like</c><00:40:03.839><c> 100</c><00:40:04.079><c> times</c>"
+    },
+    {
+      "start": 2404.309,
+      "duration": 0.0,
+      "text": "280b so in reality it's like 100 times"
+    },
+    {
+      "start": 2404.319,
+      "duration": 0.0,
+      "text": "280b so in reality it's like 100 times bigger<00:40:04.560><c> so</c><00:40:04.680><c> you</c><00:40:04.800><c> cannot</c><00:40:05.040><c> have</c><00:40:05.240><c> that</c><00:40:05.400><c> much</c><00:40:05.599><c> of</c>"
+    },
+    {
+      "start": 2405.79,
+      "duration": 0.0,
+      "text": "bigger so you cannot have that much of"
+    },
+    {
+      "start": 2405.8,
+      "duration": 0.0,
+      "text": "bigger so you cannot have that much of GitHub<00:40:06.240><c> and</c><00:40:06.480><c> and</c><00:40:06.720><c> of</c>"
+    },
+    {
+      "start": 2407.91,
+      "duration": 0.0,
+      "text": "GitHub and and of"
+    },
+    {
+      "start": 2407.92,
+      "duration": 0.0,
+      "text": "GitHub and and of Wikipedia<00:40:08.920><c> um</c><00:40:09.520><c> in</c><00:40:09.680><c> terms</c><00:40:09.880><c> of</c><00:40:10.000><c> close</c><00:40:10.319><c> Source</c>"
+    },
+    {
+      "start": 2410.55,
+      "duration": 0.0,
+      "text": "Wikipedia um in terms of close Source"
+    },
+    {
+      "start": 2410.56,
+      "duration": 0.0,
+      "text": "Wikipedia um in terms of close Source models<00:40:11.400><c> just</c><00:40:11.520><c> to</c><00:40:11.680><c> give</c><00:40:11.800><c> you</c><00:40:11.920><c> an</c><00:40:12.079><c> idea</c><00:40:12.880><c> uh</c><00:40:13.040><c> Lama</c>"
+    },
+    {
+      "start": 2413.47,
+      "duration": 0.0,
+      "text": "models just to give you an idea uh Lama"
+    },
+    {
+      "start": 2413.48,
+      "duration": 0.0,
+      "text": "models just to give you an idea uh Lama 2<00:40:14.359><c> um</c><00:40:14.680><c> it</c><00:40:14.800><c> was</c><00:40:14.960><c> trained</c><00:40:15.240><c> on</c><00:40:15.400><c> 20</c><00:40:15.760><c> two</c><00:40:16.000><c> trillion</c>"
+    },
+    {
+      "start": 2416.349,
+      "duration": 0.0,
+      "text": "2 um it was trained on 20 two trillion"
+    },
+    {
+      "start": 2416.359,
+      "duration": 0.0,
+      "text": "2 um it was trained on 20 two trillion tokens<00:40:16.920><c> lamb</c><00:40:17.319><c> 3</c><00:40:17.599><c> 15</c><00:40:17.839><c> trillion</c><00:40:18.200><c> tokens</c><00:40:18.880><c> which</c>"
+    },
+    {
+      "start": 2419.03,
+      "duration": 0.0,
+      "text": "tokens lamb 3 15 trillion tokens which"
+    },
+    {
+      "start": 2419.04,
+      "duration": 0.0,
+      "text": "tokens lamb 3 15 trillion tokens which is<00:40:19.480><c> currently</c><00:40:19.880><c> the</c><00:40:20.040><c> best</c><00:40:20.359><c> model</c><00:40:20.680><c> that</c><00:40:20.800><c> we</c><00:40:20.920><c> know</c>"
+    },
+    {
+      "start": 2421.19,
+      "duration": 0.0,
+      "text": "is currently the best model that we know"
+    },
+    {
+      "start": 2421.2,
+      "duration": 0.0,
+      "text": "is currently the best model that we know on<00:40:21.440><c> how</c><00:40:21.560><c> much</c><00:40:21.680><c> it</c><00:40:21.800><c> was</c><00:40:21.920><c> trained</c><00:40:22.200><c> on</c><00:40:22.640><c> which</c><00:40:22.760><c> is</c>"
+    },
+    {
+      "start": 2422.91,
+      "duration": 0.0,
+      "text": "on how much it was trained on which is"
+    },
+    {
+      "start": 2422.92,
+      "duration": 0.0,
+      "text": "on how much it was trained on which is the<00:40:23.040><c> same</c><00:40:23.240><c> thing</c><00:40:23.400><c> as</c><00:40:23.680><c> this</c><00:40:24.079><c> the</c><00:40:24.520><c> the</c><00:40:25.280><c> the</c><00:40:25.440><c> best</c>"
+    },
+    {
+      "start": 2425.79,
+      "duration": 0.0,
+      "text": "the same thing as this the the the best"
+    },
+    {
+      "start": 2425.8,
+      "duration": 0.0,
+      "text": "the same thing as this the the the best academic<00:40:26.319><c> or</c><00:40:26.480><c> the</c><00:40:26.640><c> biggest</c><00:40:26.960><c> academic</c>"
+    },
+    {
+      "start": 2427.47,
+      "duration": 0.0,
+      "text": "academic or the biggest academic"
+    },
+    {
+      "start": 2427.48,
+      "duration": 0.0,
+      "text": "academic or the biggest academic Benchmark<00:40:27.920><c> which</c><00:40:28.000><c> is</c><00:40:28.119><c> 15</c><00:40:28.359><c> trillion</c><00:40:28.720><c> tokens</c>"
+    },
+    {
+      "start": 2429.349,
+      "duration": 0.0,
+      "text": "Benchmark which is 15 trillion tokens"
+    },
+    {
+      "start": 2429.359,
+      "duration": 0.0,
+      "text": "Benchmark which is 15 trillion tokens GPD<00:40:29.760><c> 4</c><00:40:30.000><c> we</c><00:40:30.079><c> don't</c><00:40:30.280><c> really</c><00:40:30.440><c> know</c><00:40:30.640><c> but</c><00:40:30.760><c> it's</c>"
+    },
+    {
+      "start": 2430.91,
+      "duration": 0.0,
+      "text": "GPD 4 we don't really know but it's"
+    },
+    {
+      "start": 2430.92,
+      "duration": 0.0,
+      "text": "GPD 4 we don't really know but it's probably<00:40:31.119><c> in</c><00:40:31.200><c> the</c><00:40:31.280><c> same</c><00:40:31.440><c> water</c><00:40:31.680><c> of</c><00:40:31.800><c> magnitude</c>"
+    },
+    {
+      "start": 2432.55,
+      "duration": 0.0,
+      "text": "probably in the same water of magnitude"
+    },
+    {
+      "start": 2432.56,
+      "duration": 0.0,
+      "text": "probably in the same water of magnitude or<00:40:32.720><c> it's</c><00:40:32.880><c> probably</c><00:40:33.119><c> around</c><00:40:33.359><c> that</c><00:40:33.520><c> actually</c>"
+    },
+    {
+      "start": 2433.71,
+      "duration": 0.0,
+      "text": "or it's probably around that actually"
+    },
+    {
+      "start": 2433.72,
+      "duration": 0.0,
+      "text": "or it's probably around that actually it's<00:40:33.839><c> probably</c><00:40:34.079><c> around</c><00:40:34.280><c> 13</c><00:40:35.200><c> um</c><00:40:36.040><c> from</c><00:40:36.280><c> leaks</c><00:40:36.680><c> if</c>"
+    },
+    {
+      "start": 2436.79,
+      "duration": 0.0,
+      "text": "it's probably around 13 um from leaks if"
+    },
+    {
+      "start": 2436.8,
+      "duration": 0.0,
+      "text": "it's probably around 13 um from leaks if the<00:40:36.920><c> leaks</c><00:40:37.119><c> are</c><00:40:37.440><c> true</c>"
+    },
+    {
+      "start": 2439.03,
+      "duration": 0.0,
+      "text": "the leaks are true"
+    },
+    {
+      "start": 2439.04,
+      "duration": 0.0,
+      "text": "the leaks are true um<00:40:40.319><c> great</c><00:40:41.319><c> so</c><00:40:41.760><c> scaling</c><00:40:42.240><c> laws</c><00:40:43.079><c> um</c><00:40:43.440><c> any</c><00:40:43.599><c> other</c>"
+    },
+    {
+      "start": 2443.79,
+      "duration": 0.0,
+      "text": "um great so scaling laws um any other"
+    },
+    {
+      "start": 2443.8,
+      "duration": 0.0,
+      "text": "um great so scaling laws um any other questions<00:40:44.040><c> on</c><00:40:44.200><c> Data</c><00:40:44.480><c> before</c><00:40:44.640><c> you</c><00:40:44.760><c> go</c><00:40:44.880><c> to</c>"
+    },
+    {
+      "start": 2444.99,
+      "duration": 0.0,
+      "text": "questions on Data before you go to"
+    },
+    {
+      "start": 2445.0,
+      "duration": 0.0,
+      "text": "questions on Data before you go to scaling"
+    },
+    {
+      "start": 2448.079,
+      "duration": 0.0,
+      "text": "laws<00:40:49.079><c> sorry</c><00:40:49.400><c> I</c><00:40:49.480><c> know</c><00:40:49.599><c> I'm</c><00:40:49.720><c> giving</c><00:40:49.920><c> you</c><00:40:50.040><c> a</c><00:40:50.160><c> lot</c>"
+    },
+    {
+      "start": 2450.27,
+      "duration": 0.0,
+      "text": "laws sorry I know I'm giving you a lot"
+    },
+    {
+      "start": 2450.28,
+      "duration": 0.0,
+      "text": "laws sorry I know I'm giving you a lot of<00:40:50.480><c> information</c><00:40:51.040><c> but</c><00:40:51.240><c> uh</c><00:40:51.760><c> there's</c><00:40:51.920><c> a</c><00:40:52.040><c> lot</c><00:40:52.200><c> into</c>"
+    },
+    {
+      "start": 2452.47,
+      "duration": 0.0,
+      "text": "of information but uh there's a lot into"
+    },
+    {
+      "start": 2452.48,
+      "duration": 0.0,
+      "text": "of information but uh there's a lot into training<00:40:52.800><c> at</c><00:40:52.920><c> large</c><00:40:53.200><c> language</c><00:40:53.800><c> models</c><00:40:54.800><c> great</c>"
+    },
+    {
+      "start": 2455.309,
+      "duration": 0.0,
+      "text": "training at large language models great"
+    },
+    {
+      "start": 2455.319,
+      "duration": 0.0,
+      "text": "training at large language models great scaling<00:40:56.040><c> laws</c><00:40:57.040><c> so</c><00:40:57.319><c> so</c><00:40:57.640><c> the</c><00:40:57.800><c> idea</c><00:40:58.280><c> is</c><00:40:58.400><c> that</c><00:40:58.599><c> what</c>"
+    },
+    {
+      "start": 2458.71,
+      "duration": 0.0,
+      "text": "scaling laws so so the idea is that what"
+    },
+    {
+      "start": 2458.72,
+      "duration": 0.0,
+      "text": "scaling laws so so the idea is that what people<00:40:58.960><c> saw</c><00:40:59.839><c> um</c><00:41:00.200><c> around</c><00:41:00.520><c> 2020</c><00:41:01.280><c> or</c><00:41:01.440><c> at</c><00:41:01.520><c> least</c>"
+    },
+    {
+      "start": 2461.67,
+      "duration": 0.0,
+      "text": "people saw um around 2020 or at least"
+    },
+    {
+      "start": 2461.68,
+      "duration": 0.0,
+      "text": "people saw um around 2020 or at least from<00:41:01.880><c> a</c><00:41:02.000><c> long</c><00:41:02.200><c> time</c><00:41:02.359><c> but</c><00:41:02.480><c> they've</c><00:41:02.640><c> been</c><00:41:02.839><c> able</c>"
+    },
+    {
+      "start": 2463.069,
+      "duration": 0.0,
+      "text": "from a long time but they've been able"
+    },
+    {
+      "start": 2463.079,
+      "duration": 0.0,
+      "text": "from a long time but they've been able to<00:41:03.800><c> kind</c><00:41:03.960><c> of</c><00:41:04.480><c> theoretically</c><00:41:05.280><c> show</c><00:41:05.560><c> it</c><00:41:05.920><c> or</c>"
+    },
+    {
+      "start": 2466.109,
+      "duration": 0.0,
+      "text": "to kind of theoretically show it or"
+    },
+    {
+      "start": 2466.119,
+      "duration": 0.0,
+      "text": "to kind of theoretically show it or impurely<00:41:06.560><c> show</c><00:41:06.760><c> it</c><00:41:06.920><c> since</c><00:41:07.119><c> 2020</c><00:41:07.880><c> is</c><00:41:08.000><c> that</c><00:41:08.200><c> the</c>"
+    },
+    {
+      "start": 2468.39,
+      "duration": 0.0,
+      "text": "impurely show it since 2020 is that the"
+    },
+    {
+      "start": 2468.4,
+      "duration": 0.0,
+      "text": "impurely show it since 2020 is that the more<00:41:08.599><c> data</c><00:41:08.880><c> you</c><00:41:08.960><c> train</c><00:41:09.200><c> your</c><00:41:09.319><c> models</c><00:41:09.640><c> on</c><00:41:10.079><c> and</c>"
+    },
+    {
+      "start": 2470.19,
+      "duration": 0.0,
+      "text": "more data you train your models on and"
+    },
+    {
+      "start": 2470.2,
+      "duration": 0.0,
+      "text": "more data you train your models on and the<00:41:10.280><c> larger</c><00:41:10.599><c> the</c><00:41:10.720><c> models</c><00:41:11.160><c> the</c><00:41:11.280><c> better</c><00:41:11.520><c> the</c>"
+    },
+    {
+      "start": 2471.75,
+      "duration": 0.0,
+      "text": "the larger the models the better the"
+    },
+    {
+      "start": 2471.76,
+      "duration": 0.0,
+      "text": "the larger the models the better the performance<00:41:12.760><c> this</c><00:41:12.839><c> is</c><00:41:13.000><c> actually</c><00:41:13.200><c> pretty</c>"
+    },
+    {
+      "start": 2473.47,
+      "duration": 0.0,
+      "text": "performance this is actually pretty"
+    },
+    {
+      "start": 2473.48,
+      "duration": 0.0,
+      "text": "performance this is actually pretty different<00:41:14.079><c> than</c><00:41:14.240><c> what</c><00:41:14.359><c> you've</c><00:41:14.520><c> seen</c><00:41:14.720><c> in</c><00:41:14.880><c> this</c>"
+    },
+    {
+      "start": 2475.109,
+      "duration": 0.0,
+      "text": "different than what you've seen in this"
+    },
+    {
+      "start": 2475.119,
+      "duration": 0.0,
+      "text": "different than what you've seen in this class<00:41:15.599><c> in</c><00:41:15.800><c> this</c><00:41:15.960><c> class</c><00:41:16.200><c> we</c><00:41:16.319><c> teach</c><00:41:16.520><c> you</c><00:41:16.680><c> about</c>"
+    },
+    {
+      "start": 2476.87,
+      "duration": 0.0,
+      "text": "class in this class we teach you about"
+    },
+    {
+      "start": 2476.88,
+      "duration": 0.0,
+      "text": "class in this class we teach you about overfitting<00:41:17.720><c> overfitting</c><00:41:18.400><c> doesn't</c><00:41:18.680><c> happen</c>"
+    },
+    {
+      "start": 2478.95,
+      "duration": 0.0,
+      "text": "overfitting overfitting doesn't happen"
+    },
+    {
+      "start": 2478.96,
+      "duration": 0.0,
+      "text": "overfitting overfitting doesn't happen with<00:41:19.119><c> large</c><00:41:19.359><c> language</c><00:41:19.680><c> models</c><00:41:20.599><c> uh</c><00:41:20.839><c> larger</c>"
+    },
+    {
+      "start": 2481.27,
+      "duration": 0.0,
+      "text": "with large language models uh larger"
+    },
+    {
+      "start": 2481.28,
+      "duration": 0.0,
+      "text": "with large language models uh larger models<00:41:21.880><c> better</c><00:41:22.400><c> performance</c><00:41:23.400><c> um</c><00:41:23.839><c> it's</c>"
+    },
+    {
+      "start": 2484.069,
+      "duration": 0.0,
+      "text": "models better performance um it's"
+    },
+    {
+      "start": 2484.079,
+      "duration": 0.0,
+      "text": "models better performance um it's something<00:41:24.480><c> that</c><00:41:24.680><c> really</c><00:41:24.920><c> took</c><00:41:25.119><c> a</c><00:41:25.280><c> long</c><00:41:25.560><c> time</c>"
+    },
+    {
+      "start": 2485.71,
+      "duration": 0.0,
+      "text": "something that really took a long time"
+    },
+    {
+      "start": 2485.72,
+      "duration": 0.0,
+      "text": "something that really took a long time for<00:41:25.880><c> the</c><00:41:26.040><c> community</c><00:41:26.800><c> who</c><00:41:26.960><c> took</c><00:41:27.440><c> this</c><00:41:27.599><c> type</c><00:41:27.800><c> of</c>"
+    },
+    {
+      "start": 2487.95,
+      "duration": 0.0,
+      "text": "for the community who took this type of"
+    },
+    {
+      "start": 2487.96,
+      "duration": 0.0,
+      "text": "for the community who took this type of class<00:41:28.440><c> to</c><00:41:28.800><c> realize</c><00:41:29.800><c> um</c><00:41:30.160><c> but</c><00:41:30.319><c> for</c><00:41:30.440><c> the</c><00:41:30.599><c> exam</c>"
+    },
+    {
+      "start": 2491.27,
+      "duration": 0.0,
+      "text": "class to realize um but for the exam"
+    },
+    {
+      "start": 2491.28,
+      "duration": 0.0,
+      "text": "class to realize um but for the exam overfitting"
+    },
+    {
+      "start": 2492.75,
+      "duration": 0.0,
+      "text": "overfitting"
+    },
+    {
+      "start": 2492.76,
+      "duration": 0.0,
+      "text": "overfitting exists<00:41:33.760><c> so</c><00:41:34.680><c> okay</c><00:41:35.040><c> the</c><00:41:35.240><c> idea</c><00:41:35.800><c> of</c><00:41:35.920><c> scaling</c><00:41:36.280><c> laws</c>"
+    },
+    {
+      "start": 2496.75,
+      "duration": 0.0,
+      "text": "exists so okay the idea of scaling laws"
+    },
+    {
+      "start": 2496.76,
+      "duration": 0.0,
+      "text": "exists so okay the idea of scaling laws is<00:41:36.880><c> that</c><00:41:37.240><c> if</c><00:41:37.599><c> given</c><00:41:37.839><c> that</c><00:41:37.960><c> you</c><00:41:38.040><c> know</c><00:41:38.240><c> that</c><00:41:38.359><c> more</c>"
+    },
+    {
+      "start": 2498.589,
+      "duration": 0.0,
+      "text": "is that if given that you know that more"
+    },
+    {
+      "start": 2498.599,
+      "duration": 0.0,
+      "text": "is that if given that you know that more data<00:41:39.200><c> and</c><00:41:39.359><c> larger</c><00:41:40.000><c> models</c><00:41:40.440><c> will</c><00:41:40.680><c> always</c><00:41:41.040><c> give</c>"
+    },
+    {
+      "start": 2501.15,
+      "duration": 0.0,
+      "text": "data and larger models will always give"
+    },
+    {
+      "start": 2501.16,
+      "duration": 0.0,
+      "text": "data and larger models will always give you<00:41:41.319><c> better</c><00:41:41.680><c> performance</c><00:41:42.680><c> can</c><00:41:42.839><c> we</c><00:41:43.160><c> predict</c>"
+    },
+    {
+      "start": 2504.15,
+      "duration": 0.0,
+      "text": "you better performance can we predict"
+    },
+    {
+      "start": 2504.16,
+      "duration": 0.0,
+      "text": "you better performance can we predict how<00:41:44.760><c> much</c><00:41:45.040><c> better</c><00:41:45.319><c> your</c><00:41:45.480><c> performance</c><00:41:46.000><c> will</c><00:41:46.200><c> be</c>"
+    },
+    {
+      "start": 2506.67,
+      "duration": 0.0,
+      "text": "how much better your performance will be"
+    },
+    {
+      "start": 2506.68,
+      "duration": 0.0,
+      "text": "how much better your performance will be if<00:41:46.800><c> you</c><00:41:47.000><c> increase</c><00:41:47.480><c> the</c><00:41:47.599><c> amount</c><00:41:47.800><c> of</c><00:41:47.960><c> data</c><00:41:48.280><c> and</c>"
+    },
+    {
+      "start": 2508.39,
+      "duration": 0.0,
+      "text": "if you increase the amount of data and"
+    },
+    {
+      "start": 2508.4,
+      "duration": 0.0,
+      "text": "if you increase the amount of data and the<00:41:48.560><c> size</c><00:41:48.760><c> of</c><00:41:48.880><c> your</c><00:41:49.440><c> model</c><00:41:50.440><c> and</c><00:41:50.599><c> surprisingly</c>"
+    },
+    {
+      "start": 2511.27,
+      "duration": 0.0,
+      "text": "the size of your model and surprisingly"
+    },
+    {
+      "start": 2511.28,
+      "duration": 0.0,
+      "text": "the size of your model and surprisingly it<00:41:51.520><c> works</c><00:41:52.520><c> uh</c><00:41:52.640><c> so</c><00:41:52.839><c> here</c><00:41:52.960><c> you</c><00:41:53.079><c> see</c><00:41:53.359><c> three</c><00:41:53.560><c> plots</c>"
+    },
+    {
+      "start": 2513.87,
+      "duration": 0.0,
+      "text": "it works uh so here you see three plots"
+    },
+    {
+      "start": 2513.88,
+      "duration": 0.0,
+      "text": "it works uh so here you see three plots from<00:41:54.040><c> a</c><00:41:54.200><c> very</c><00:41:54.400><c> famous</c><00:41:54.720><c> paper</c><00:41:55.079><c> called</c><00:41:55.280><c> scaling</c>"
+    },
+    {
+      "start": 2515.63,
+      "duration": 0.0,
+      "text": "from a very famous paper called scaling"
+    },
+    {
+      "start": 2515.64,
+      "duration": 0.0,
+      "text": "from a very famous paper called scaling loss<00:41:56.000><c> from</c><00:41:56.160><c> openi</c><00:41:57.359><c> um</c><00:41:57.960><c> here</c><00:41:58.079><c> you</c><00:41:58.200><c> see</c><00:41:58.400><c> on</c><00:41:58.520><c> the</c>"
+    },
+    {
+      "start": 2518.67,
+      "duration": 0.0,
+      "text": "loss from openi um here you see on the"
+    },
+    {
+      "start": 2518.68,
+      "duration": 0.0,
+      "text": "loss from openi um here you see on the x-axis<00:41:59.319><c> compute</c><00:42:00.000><c> so</c><00:42:00.359><c> how</c><00:42:00.520><c> much</c><00:42:00.680><c> did</c><00:42:00.839><c> you</c><00:42:00.960><c> train</c>"
+    },
+    {
+      "start": 2521.589,
+      "duration": 0.0,
+      "text": "x-axis compute so how much did you train"
+    },
+    {
+      "start": 2521.599,
+      "duration": 0.0,
+      "text": "x-axis compute so how much did you train like<00:42:01.760><c> how</c><00:42:01.839><c> much</c><00:42:02.040><c> compute</c><00:42:02.359><c> did</c><00:42:02.520><c> you</c><00:42:02.720><c> did</c><00:42:02.839><c> you</c>"
+    },
+    {
+      "start": 2522.91,
+      "duration": 0.0,
+      "text": "like how much compute did you did you"
+    },
+    {
+      "start": 2522.92,
+      "duration": 0.0,
+      "text": "like how much compute did you did you spend<00:42:03.160><c> for</c><00:42:03.319><c> training</c><00:42:03.960><c> and</c><00:42:04.160><c> here</c><00:42:04.280><c> you</c><00:42:04.400><c> see</c><00:42:04.599><c> test</c>"
+    },
+    {
+      "start": 2524.87,
+      "duration": 0.0,
+      "text": "spend for training and here you see test"
+    },
+    {
+      "start": 2524.88,
+      "duration": 0.0,
+      "text": "spend for training and here you see test loss<00:42:05.319><c> so</c><00:42:05.520><c> this</c><00:42:05.640><c> is</c><00:42:05.920><c> essentially</c><00:42:06.880><c> I</c><00:42:06.920><c> mean</c><00:42:07.040><c> it's</c>"
+    },
+    {
+      "start": 2527.15,
+      "duration": 0.0,
+      "text": "loss so this is essentially I mean it's"
+    },
+    {
+      "start": 2527.16,
+      "duration": 0.0,
+      "text": "loss so this is essentially I mean it's not<00:42:07.319><c> perplexity</c><00:42:07.839><c> but</c><00:42:07.920><c> it's</c><00:42:08.000><c> your</c><00:42:08.160><c> validation</c>"
+    },
+    {
+      "start": 2528.589,
+      "duration": 0.0,
+      "text": "not perplexity but it's your validation"
+    },
+    {
+      "start": 2528.599,
+      "duration": 0.0,
+      "text": "not perplexity but it's your validation loss<00:42:09.319><c> um</c><00:42:09.640><c> so</c><00:42:09.920><c> it's</c><00:42:10.040><c> a</c><00:42:10.160><c> log</c><00:42:10.400><c> of</c><00:42:10.480><c> the</c><00:42:10.640><c> perplexity</c>"
+    },
+    {
+      "start": 2531.63,
+      "duration": 0.0,
+      "text": "loss um so it's a log of the perplexity"
+    },
+    {
+      "start": 2531.64,
+      "duration": 0.0,
+      "text": "loss um so it's a log of the perplexity and<00:42:11.839><c> if</c><00:42:11.920><c> you</c><00:42:12.119><c> put</c><00:42:12.400><c> these</c><00:42:12.560><c> two</c><00:42:13.200><c> on</c><00:42:13.720><c> uh</c><00:42:13.839><c> log</c><00:42:14.160><c> scale</c>"
+    },
+    {
+      "start": 2535.069,
+      "duration": 0.0,
+      "text": "and if you put these two on uh log scale"
+    },
+    {
+      "start": 2535.079,
+      "duration": 0.0,
+      "text": "and if you put these two on uh log scale uh<00:42:15.200><c> then</c><00:42:15.319><c> you</c><00:42:15.440><c> see</c><00:42:15.760><c> that</c><00:42:16.200><c> uh</c><00:42:16.440><c> the</c><00:42:17.000><c> the</c>"
+    },
+    {
+      "start": 2537.15,
+      "duration": 0.0,
+      "text": "uh then you see that uh the the"
+    },
+    {
+      "start": 2537.16,
+      "duration": 0.0,
+      "text": "uh then you see that uh the the performance<00:42:17.920><c> or</c><00:42:18.079><c> like</c><00:42:18.280><c> the</c><00:42:18.480><c> this</c><00:42:18.960><c> the</c><00:42:19.680><c> sorry</c>"
+    },
+    {
+      "start": 2540.109,
+      "duration": 0.0,
+      "text": "performance or like the this the sorry"
+    },
+    {
+      "start": 2540.119,
+      "duration": 0.0,
+      "text": "performance or like the this the sorry the<00:42:20.640><c> the</c><00:42:20.720><c> scaling</c><00:42:21.160><c> law</c><00:42:21.400><c> is</c><00:42:21.559><c> linear</c><00:42:22.480><c> uh</c><00:42:22.640><c> that</c>"
+    },
+    {
+      "start": 2542.79,
+      "duration": 0.0,
+      "text": "the the scaling law is linear uh that"
+    },
+    {
+      "start": 2542.8,
+      "duration": 0.0,
+      "text": "the the scaling law is linear uh that means<00:42:23.160><c> that</c><00:42:23.440><c> if</c><00:42:23.559><c> you</c><00:42:23.720><c> increase</c><00:42:24.119><c> your</c><00:42:24.319><c> compute</c>"
+    },
+    {
+      "start": 2545.069,
+      "duration": 0.0,
+      "text": "means that if you increase your compute"
+    },
+    {
+      "start": 2545.079,
+      "duration": 0.0,
+      "text": "means that if you increase your compute by<00:42:25.200><c> a</c><00:42:25.319><c> certain</c><00:42:25.599><c> amount</c><00:42:25.839><c> you</c><00:42:26.000><c> can</c><00:42:26.319><c> you</c><00:42:26.400><c> can</c><00:42:26.559><c> say</c>"
+    },
+    {
+      "start": 2546.75,
+      "duration": 0.0,
+      "text": "by a certain amount you can you can say"
+    },
+    {
+      "start": 2546.76,
+      "duration": 0.0,
+      "text": "by a certain amount you can you can say by<00:42:26.880><c> how</c><00:42:27.280><c> much</c><00:42:27.760><c> your</c><00:42:28.040><c> test</c><00:42:28.319><c> loss</c><00:42:28.760><c> will</c><00:42:29.000><c> actually</c>"
+    },
+    {
+      "start": 2549.47,
+      "duration": 0.0,
+      "text": "by how much your test loss will actually"
+    },
+    {
+      "start": 2549.48,
+      "duration": 0.0,
+      "text": "by how much your test loss will actually decrease<00:42:30.480><c> same</c><00:42:30.720><c> thing</c><00:42:30.880><c> with</c><00:42:31.119><c> data</c><00:42:31.760><c> and</c><00:42:31.920><c> same</c>"
+    },
+    {
+      "start": 2552.109,
+      "duration": 0.0,
+      "text": "decrease same thing with data and same"
+    },
+    {
+      "start": 2552.119,
+      "duration": 0.0,
+      "text": "decrease same thing with data and same thing<00:42:32.240><c> for</c><00:42:32.559><c> parameters</c><00:42:33.440><c> if</c><00:42:33.559><c> you</c><00:42:33.760><c> increase</c><00:42:34.160><c> the</c>"
+    },
+    {
+      "start": 2554.27,
+      "duration": 0.0,
+      "text": "thing for parameters if you increase the"
+    },
+    {
+      "start": 2554.28,
+      "duration": 0.0,
+      "text": "thing for parameters if you increase the data<00:42:34.559><c> set</c><00:42:34.800><c> size</c><00:42:35.480><c> your</c><00:42:35.680><c> loss</c><00:42:36.040><c> will</c><00:42:36.480><c> will</c>"
+    },
+    {
+      "start": 2556.67,
+      "duration": 0.0,
+      "text": "data set size your loss will will"
+    },
+    {
+      "start": 2556.68,
+      "duration": 0.0,
+      "text": "data set size your loss will will decrease<00:42:37.480><c> by</c><00:42:37.720><c> an</c><00:42:37.960><c> amount</c><00:42:38.480><c> that</c><00:42:38.720><c> is</c><00:42:38.920><c> somewhat</c>"
+    },
+    {
+      "start": 2559.27,
+      "duration": 0.0,
+      "text": "decrease by an amount that is somewhat"
+    },
+    {
+      "start": 2559.28,
+      "duration": 0.0,
+      "text": "decrease by an amount that is somewhat predictable<00:42:40.040><c> if</c><00:42:40.160><c> you</c><00:42:40.319><c> increase</c><00:42:40.640><c> the</c><00:42:40.760><c> number</c>"
+    },
+    {
+      "start": 2560.95,
+      "duration": 0.0,
+      "text": "predictable if you increase the number"
+    },
+    {
+      "start": 2560.96,
+      "duration": 0.0,
+      "text": "predictable if you increase the number of<00:42:41.040><c> parameters</c><00:42:42.000><c> it</c><00:42:42.119><c> will</c><00:42:42.359><c> decre</c><00:42:42.720><c> the</c><00:42:42.800><c> loss</c>"
+    },
+    {
+      "start": 2563.03,
+      "duration": 0.0,
+      "text": "of parameters it will decre the loss"
+    },
+    {
+      "start": 2563.04,
+      "duration": 0.0,
+      "text": "of parameters it will decre the loss will<00:42:43.200><c> decrease</c><00:42:43.599><c> by</c><00:42:43.960><c> amount</c><00:42:44.280><c> which</c><00:42:44.359><c> is</c>"
+    },
+    {
+      "start": 2564.47,
+      "duration": 0.0,
+      "text": "will decrease by amount which is"
+    },
+    {
+      "start": 2564.48,
+      "duration": 0.0,
+      "text": "will decrease by amount which is somewhat<00:42:44.800><c> predictable</c><00:42:45.760><c> this</c><00:42:45.880><c> is</c><00:42:46.240><c> really</c>"
+    },
+    {
+      "start": 2566.589,
+      "duration": 0.0,
+      "text": "somewhat predictable this is really"
+    },
+    {
+      "start": 2566.599,
+      "duration": 0.0,
+      "text": "somewhat predictable this is really amazing<00:42:47.599><c> um</c><00:42:48.240><c> very</c><00:42:48.520><c> surprising</c><00:42:49.520><c> I</c><00:42:49.599><c> mean</c><00:42:49.760><c> it</c>"
+    },
+    {
+      "start": 2569.87,
+      "duration": 0.0,
+      "text": "amazing um very surprising I mean it"
+    },
+    {
+      "start": 2569.88,
+      "duration": 0.0,
+      "text": "amazing um very surprising I mean it looks<00:42:50.319><c> in</c><00:42:50.520><c> nocuous</c><00:42:51.040><c> when</c><00:42:51.200><c> you</c><00:42:51.359><c> look</c><00:42:51.480><c> at</c><00:42:51.640><c> these</c>"
+    },
+    {
+      "start": 2571.79,
+      "duration": 0.0,
+      "text": "looks in nocuous when you look at these"
+    },
+    {
+      "start": 2571.8,
+      "duration": 0.0,
+      "text": "looks in nocuous when you look at these type<00:42:52.000><c> of</c><00:42:52.119><c> plots</c><00:42:52.640><c> but</c><00:42:52.760><c> that's</c><00:42:52.960><c> crazy</c><00:42:53.319><c> because</c>"
+    },
+    {
+      "start": 2573.43,
+      "duration": 0.0,
+      "text": "type of plots but that's crazy because"
+    },
+    {
+      "start": 2573.44,
+      "duration": 0.0,
+      "text": "type of plots but that's crazy because it<00:42:53.520><c> means</c><00:42:53.720><c> that</c><00:42:53.839><c> you</c><00:42:53.920><c> can</c><00:42:54.119><c> predict</c><00:42:55.119><c> uh</c><00:42:55.359><c> how</c>"
+    },
+    {
+      "start": 2575.549,
+      "duration": 0.0,
+      "text": "it means that you can predict uh how"
+    },
+    {
+      "start": 2575.559,
+      "duration": 0.0,
+      "text": "it means that you can predict uh how well<00:42:55.760><c> we're</c><00:42:55.920><c> going</c><00:42:56.000><c> to</c><00:42:56.160><c> perform</c><00:42:56.800><c> in</c><00:42:57.160><c> 2</c><00:42:57.400><c> 3</c><00:42:57.640><c> years</c>"
+    },
+    {
+      "start": 2578.069,
+      "duration": 0.0,
+      "text": "well we're going to perform in 2 3 years"
+    },
+    {
+      "start": 2578.079,
+      "duration": 0.0,
+      "text": "well we're going to perform in 2 3 years depending<00:42:58.400><c> on</c><00:42:58.559><c> how</c><00:42:58.680><c> much</c><00:42:58.880><c> compute</c><00:42:59.240><c> we</c><00:42:59.359><c> will</c>"
+    },
+    {
+      "start": 2579.549,
+      "duration": 0.0,
+      "text": "depending on how much compute we will"
+    },
+    {
+      "start": 2579.559,
+      "duration": 0.0,
+      "text": "depending on how much compute we will add<00:43:00.000><c> assuming</c><00:43:00.400><c> that</c><00:43:00.559><c> these</c><00:43:00.720><c> things</c><00:43:01.000><c> will</c><00:43:01.200><c> hold</c>"
+    },
+    {
+      "start": 2581.51,
+      "duration": 0.0,
+      "text": "add assuming that these things will hold"
+    },
+    {
+      "start": 2581.52,
+      "duration": 0.0,
+      "text": "add assuming that these things will hold there's<00:43:01.680><c> nothing</c><00:43:01.920><c> theoretical</c><00:43:02.440><c> about</c><00:43:02.640><c> it</c><00:43:03.599><c> um</c>"
+    },
+    {
+      "start": 2584.79,
+      "duration": 0.0,
+      "text": "there's nothing theoretical about it um"
+    },
+    {
+      "start": 2584.8,
+      "duration": 0.0,
+      "text": "there's nothing theoretical about it um yes<00:43:05.800><c> two</c><00:43:06.040><c> things</c><00:43:06.520><c> one</c><00:43:06.839><c> what</c><00:43:06.960><c> is</c><00:43:07.119><c> the</c><00:43:07.240><c> loss</c><00:43:07.480><c> that</c>"
+    },
+    {
+      "start": 2587.589,
+      "duration": 0.0,
+      "text": "yes two things one what is the loss that"
+    },
+    {
+      "start": 2587.599,
+      "duration": 0.0,
+      "text": "yes two things one what is the loss that they're<00:43:07.720><c> using</c><00:43:07.960><c> here</c><00:43:08.079><c> is</c><00:43:08.200><c> this</c><00:43:08.400><c> perplexity</c><00:43:09.079><c> or</c>"
+    },
+    {
+      "start": 2589.51,
+      "duration": 0.0,
+      "text": "they're using here is this perplexity or"
+    },
+    {
+      "start": 2589.52,
+      "duration": 0.0,
+      "text": "they're using here is this perplexity or so<00:43:09.680><c> it's</c><00:43:10.000><c> it's</c><00:43:10.440><c> you</c><00:43:10.559><c> know</c><00:43:10.760><c> I</c><00:43:10.839><c> said</c><00:43:11.040><c> perplexity</c>"
+    },
+    {
+      "start": 2591.549,
+      "duration": 0.0,
+      "text": "so it's it's you know I said perplexity"
+    },
+    {
+      "start": 2591.559,
+      "duration": 0.0,
+      "text": "so it's it's you know I said perplexity was<00:43:11.720><c> like</c><00:43:11.880><c> two</c><00:43:12.119><c> to</c><00:43:12.280><c> the</c><00:43:12.400><c> power</c><00:43:12.680><c> of</c><00:43:12.880><c> the</c><00:43:13.000><c> LW</c><00:43:13.319><c> so</c>"
+    },
+    {
+      "start": 2593.51,
+      "duration": 0.0,
+      "text": "was like two to the power of the LW so"
+    },
+    {
+      "start": 2593.52,
+      "duration": 0.0,
+      "text": "was like two to the power of the LW so this<00:43:13.599><c> is</c><00:43:13.880><c> the</c><00:43:14.559><c> the</c><00:43:14.920><c> the</c><00:43:15.079><c> power</c><00:43:15.880><c> of</c><00:43:16.040><c> the</c>"
+    },
+    {
+      "start": 2596.15,
+      "duration": 0.0,
+      "text": "this is the the the power of the"
+    },
+    {
+      "start": 2596.16,
+      "duration": 0.0,
+      "text": "this is the the the power of the perplexity<00:43:17.000><c> and</c><00:43:17.119><c> then</c><00:43:17.400><c> the</c><00:43:17.559><c> second</c><00:43:17.880><c> thing</c><00:43:18.280><c> is</c>"
+    },
+    {
+      "start": 2598.87,
+      "duration": 0.0,
+      "text": "perplexity and then the second thing is"
+    },
+    {
+      "start": 2598.88,
+      "duration": 0.0,
+      "text": "perplexity and then the second thing is when<00:43:19.040><c> you</c><00:43:19.520><c> like</c><00:43:19.680><c> increase</c><00:43:20.040><c> the</c><00:43:20.119><c> number</c><00:43:20.319><c> of</c>"
+    },
+    {
+      "start": 2600.47,
+      "duration": 0.0,
+      "text": "when you like increase the number of"
+    },
+    {
+      "start": 2600.48,
+      "duration": 0.0,
+      "text": "when you like increase the number of parameters<00:43:20.960><c> or</c><00:43:21.079><c> you</c><00:43:21.240><c> increase</c><00:43:21.520><c> the</c><00:43:21.640><c> total</c>"
+    },
+    {
+      "start": 2601.95,
+      "duration": 0.0,
+      "text": "parameters or you increase the total"
+    },
+    {
+      "start": 2601.96,
+      "duration": 0.0,
+      "text": "parameters or you increase the total data<00:43:22.200><c> set</c><00:43:22.400><c> size</c><00:43:22.839><c> going</c><00:43:23.599><c> dat</c><00:43:24.599><c> times</c><00:43:25.000><c> doesn't</c>"
+    },
+    {
+      "start": 2605.27,
+      "duration": 0.0,
+      "text": "data set size going dat times doesn't"
+    },
+    {
+      "start": 2605.28,
+      "duration": 0.0,
+      "text": "data set size going dat times doesn't that<00:43:25.480><c> just</c><00:43:26.200><c> inherently</c><00:43:26.680><c> increase</c><00:43:26.960><c> your</c>"
+    },
+    {
+      "start": 2607.23,
+      "duration": 0.0,
+      "text": "that just inherently increase your"
+    },
+    {
+      "start": 2607.24,
+      "duration": 0.0,
+      "text": "that just inherently increase your compute<00:43:27.720><c> like</c><00:43:28.000><c> do</c><00:43:28.160><c> all</c><00:43:28.400><c> this</c><00:43:28.559><c> work</c><00:43:29.040><c> to</c>"
+    },
+    {
+      "start": 2611.069,
+      "duration": 0.0,
+      "text": "compute like do all this work to"
+    },
+    {
+      "start": 2611.079,
+      "duration": 0.0,
+      "text": "compute like do all this work to just<00:43:32.079><c> specific</c><00:43:32.640><c> no</c><00:43:32.760><c> this</c><00:43:32.839><c> is</c><00:43:32.920><c> a</c><00:43:33.040><c> great</c>"
+    },
+    {
+      "start": 2613.23,
+      "duration": 0.0,
+      "text": "just specific no this is a great"
+    },
+    {
+      "start": 2613.24,
+      "duration": 0.0,
+      "text": "just specific no this is a great question<00:43:33.640><c> so</c><00:43:33.800><c> the</c><00:43:33.960><c> compute</c><00:43:34.480><c> here</c><00:43:35.079><c> is</c><00:43:35.240><c> actually</c>"
+    },
+    {
+      "start": 2615.39,
+      "duration": 0.0,
+      "text": "question so the compute here is actually"
+    },
+    {
+      "start": 2615.4,
+      "duration": 0.0,
+      "text": "question so the compute here is actually a<00:43:35.559><c> factor</c><00:43:35.839><c> of</c><00:43:35.960><c> two</c><00:43:36.160><c> things</c><00:43:36.559><c> the</c><00:43:36.760><c> data</c><00:43:37.200><c> and</c><00:43:37.359><c> the</c>"
+    },
+    {
+      "start": 2617.51,
+      "duration": 0.0,
+      "text": "a factor of two things the data and the"
+    },
+    {
+      "start": 2617.52,
+      "duration": 0.0,
+      "text": "a factor of two things the data and the parameter<00:43:38.359><c> what</c><00:43:38.480><c> I'm</c><00:43:38.599><c> showing</c><00:43:38.960><c> here</c><00:43:39.160><c> is</c><00:43:39.280><c> that</c>"
+    },
+    {
+      "start": 2619.43,
+      "duration": 0.0,
+      "text": "parameter what I'm showing here is that"
+    },
+    {
+      "start": 2619.44,
+      "duration": 0.0,
+      "text": "parameter what I'm showing here is that you<00:43:39.599><c> can</c><00:43:40.079><c> um</c><00:43:40.319><c> well</c><00:43:40.480><c> actually</c><00:43:40.640><c> we're</c><00:43:40.760><c> going</c><00:43:40.839><c> to</c>"
+    },
+    {
+      "start": 2620.95,
+      "duration": 0.0,
+      "text": "you can um well actually we're going to"
+    },
+    {
+      "start": 2620.96,
+      "duration": 0.0,
+      "text": "you can um well actually we're going to talk<00:43:41.119><c> about</c><00:43:41.319><c> that</c><00:43:41.440><c> in</c><00:43:41.599><c> details</c><00:43:42.040><c> but</c><00:43:42.200><c> basically</c>"
+    },
+    {
+      "start": 2622.91,
+      "duration": 0.0,
+      "text": "talk about that in details but basically"
+    },
+    {
+      "start": 2622.92,
+      "duration": 0.0,
+      "text": "talk about that in details but basically if<00:43:43.079><c> you</c><00:43:43.200><c> increase</c><00:43:43.480><c> the</c><00:43:43.559><c> number</c><00:43:43.720><c> of</c><00:43:43.839><c> parameters</c>"
+    },
+    {
+      "start": 2624.43,
+      "duration": 0.0,
+      "text": "if you increase the number of parameters"
+    },
+    {
+      "start": 2624.44,
+      "duration": 0.0,
+      "text": "if you increase the number of parameters you<00:43:44.520><c> should</c><00:43:44.720><c> increase</c><00:43:45.040><c> the</c><00:43:45.119><c> number</c><00:43:45.319><c> of</c><00:43:45.520><c> data</c>"
+    },
+    {
+      "start": 2625.99,
+      "duration": 0.0,
+      "text": "you should increase the number of data"
+    },
+    {
+      "start": 2626.0,
+      "duration": 0.0,
+      "text": "you should increase the number of data that<00:43:46.160><c> you</c><00:43:46.440><c> have</c><00:43:47.480><c> um</c><00:43:48.480><c> so</c><00:43:48.720><c> you</c><00:43:48.880><c> actually</c><00:43:49.040><c> don't</c>"
+    },
+    {
+      "start": 2629.23,
+      "duration": 0.0,
+      "text": "that you have um so you actually don't"
+    },
+    {
+      "start": 2629.24,
+      "duration": 0.0,
+      "text": "that you have um so you actually don't go<00:43:49.440><c> multiple</c><00:43:49.800><c> times</c><00:43:50.040><c> through</c><00:43:50.200><c> the</c><00:43:50.319><c> same</c><00:43:50.480><c> data</c>"
+    },
+    {
+      "start": 2630.75,
+      "duration": 0.0,
+      "text": "go multiple times through the same data"
+    },
+    {
+      "start": 2630.76,
+      "duration": 0.0,
+      "text": "go multiple times through the same data set<00:43:51.559><c> no</c><00:43:51.680><c> one</c><00:43:51.960><c> does</c><00:43:52.480><c> EPO</c><00:43:53.480><c> in</c><00:43:54.079><c> a</c><00:43:54.240><c> lar</c><00:43:54.920><c> at</c><00:43:55.000><c> least</c>"
+    },
+    {
+      "start": 2635.23,
+      "duration": 0.0,
+      "text": "set no one does EPO in a lar at least"
+    },
+    {
+      "start": 2635.24,
+      "duration": 0.0,
+      "text": "set no one does EPO in a lar at least not<00:43:55.640><c> yet</c><00:43:56.640><c> uh</c><00:43:56.760><c> because</c><00:43:57.119><c> we</c><00:43:57.240><c> have</c><00:43:57.640><c> still</c><00:43:58.359><c> kind</c><00:43:58.520><c> of</c>"
+    },
+    {
+      "start": 2638.67,
+      "duration": 0.0,
+      "text": "not yet uh because we have still kind of"
+    },
+    {
+      "start": 2638.68,
+      "duration": 0.0,
+      "text": "not yet uh because we have still kind of enough<00:43:59.000><c> data</c><00:43:59.800><c> um</c><00:44:00.079><c> so</c><00:44:00.319><c> yeah</c><00:44:00.480><c> this</c><00:44:00.599><c> is</c><00:44:00.760><c> all</c><00:44:00.960><c> the</c>"
+    },
+    {
+      "start": 2641.109,
+      "duration": 0.0,
+      "text": "enough data um so yeah this is all the"
+    },
+    {
+      "start": 2641.119,
+      "duration": 0.0,
+      "text": "enough data um so yeah this is all the same<00:44:01.359><c> Trend</c><00:44:01.720><c> which</c><00:44:01.839><c> is</c><00:44:02.040><c> increase</c><00:44:02.480><c> compute</c>"
+    },
+    {
+      "start": 2643.069,
+      "duration": 0.0,
+      "text": "same Trend which is increase compute"
+    },
+    {
+      "start": 2643.079,
+      "duration": 0.0,
+      "text": "same Trend which is increase compute decrease"
+    },
+    {
+      "start": 2644.19,
+      "duration": 0.0,
+      "text": "decrease"
+    },
+    {
+      "start": 2644.2,
+      "duration": 0.0,
+      "text": "decrease loss<00:44:05.200><c> yes</c><00:44:06.040><c> have</c><00:44:06.200><c> we</c><00:44:06.319><c> seen</c><00:44:06.720><c> the</c><00:44:06.920><c> numbers</c><00:44:07.280><c> for</c>"
+    },
+    {
+      "start": 2647.51,
+      "duration": 0.0,
+      "text": "loss yes have we seen the numbers for"
+    },
+    {
+      "start": 2647.52,
+      "duration": 0.0,
+      "text": "loss yes have we seen the numbers for the<00:44:07.720><c> last</c><00:44:08.000><c> two</c><00:44:08.240><c> years</c><00:44:09.040><c> or</c><00:44:09.480><c> is</c><00:44:09.640><c> it</c><00:44:09.839><c> still</c>"
+    },
+    {
+      "start": 2650.109,
+      "duration": 0.0,
+      "text": "the last two years or is it still"
+    },
+    {
+      "start": 2650.119,
+      "duration": 0.0,
+      "text": "the last two years or is it still holding<00:44:11.040><c> it</c><00:44:11.160><c> is</c><00:44:11.359><c> still</c><00:44:11.680><c> holding</c><00:44:12.680><c> I</c><00:44:13.520><c> I</c><00:44:13.640><c> don't</c>"
+    },
+    {
+      "start": 2653.99,
+      "duration": 0.0,
+      "text": "holding it is still holding I I don't"
+    },
+    {
+      "start": 2654.0,
+      "duration": 0.0,
+      "text": "holding it is still holding I I don't have<00:44:14.280><c> like</c><00:44:14.520><c> good</c><00:44:14.839><c> numbers</c><00:44:15.240><c> to</c><00:44:15.400><c> show</c><00:44:15.640><c> you</c><00:44:16.480><c> uh</c>"
+    },
+    {
+      "start": 2656.63,
+      "duration": 0.0,
+      "text": "have like good numbers to show you uh"
+    },
+    {
+      "start": 2656.64,
+      "duration": 0.0,
+      "text": "have like good numbers to show you uh but<00:44:16.760><c> it</c><00:44:16.880><c> is</c><00:44:17.079><c> still</c><00:44:17.319><c> holding</c>"
+    },
+    {
+      "start": 2660.28,
+      "duration": 0.0,
+      "text": "surprisingly<00:44:21.280><c> yes</c><00:44:21.800><c> is</c><00:44:21.920><c> there</c><00:44:22.160><c> no</c><00:44:22.359><c> evidence</c>"
+    },
+    {
+      "start": 2662.79,
+      "duration": 0.0,
+      "text": "surprisingly yes is there no evidence"
+    },
+    {
+      "start": 2662.8,
+      "duration": 0.0,
+      "text": "surprisingly yes is there no evidence like<00:44:22.920><c> empirical</c><00:44:23.359><c> evidence</c><00:44:23.640><c> that</c><00:44:23.720><c> you</c>"
+    },
+    {
+      "start": 2665.27,
+      "duration": 0.0,
+      "text": "like empirical evidence that you"
+    },
+    {
+      "start": 2665.28,
+      "duration": 0.0,
+      "text": "like empirical evidence that you plateau<00:44:26.280><c> expected</c><00:44:26.640><c> PL</c>"
+    },
+    {
+      "start": 2668.87,
+      "duration": 0.0,
+      "text": "plateau expected PL"
+    },
+    {
+      "start": 2668.88,
+      "duration": 0.0,
+      "text": "plateau expected PL no<00:44:29.160><c> empirical</c><00:44:29.680><c> evidence</c><00:44:29.960><c> of</c><00:44:30.119><c> plateauing</c>"
+    },
+    {
+      "start": 2670.589,
+      "duration": 0.0,
+      "text": "no empirical evidence of plateauing"
+    },
+    {
+      "start": 2670.599,
+      "duration": 0.0,
+      "text": "no empirical evidence of plateauing anytime<00:44:31.359><c> soon</c><00:44:32.480><c> um</c><00:44:33.480><c> why</c><00:44:34.319><c> we</c><00:44:34.440><c> don't</c><00:44:34.720><c> know</c><00:44:35.720><c> um</c>"
+    },
+    {
+      "start": 2676.15,
+      "duration": 0.0,
+      "text": "anytime soon um why we don't know um"
+    },
+    {
+      "start": 2676.16,
+      "duration": 0.0,
+      "text": "anytime soon um why we don't know um will<00:44:36.359><c> it</c><00:44:36.720><c> happen</c><00:44:37.720><c> probably</c><00:44:38.280><c> I</c><00:44:38.319><c> mean</c><00:44:38.480><c> it</c>"
+    },
+    {
+      "start": 2678.549,
+      "duration": 0.0,
+      "text": "will it happen probably I mean it"
+    },
+    {
+      "start": 2678.559,
+      "duration": 0.0,
+      "text": "will it happen probably I mean it doesn't<00:44:38.760><c> need</c><00:44:38.920><c> to</c><00:44:39.160><c> because</c><00:44:39.319><c> it's</c><00:44:39.440><c> actually</c><00:44:39.599><c> in</c>"
+    },
+    {
+      "start": 2679.75,
+      "duration": 0.0,
+      "text": "doesn't need to because it's actually in"
+    },
+    {
+      "start": 2679.76,
+      "duration": 0.0,
+      "text": "doesn't need to because it's actually in log<00:44:40.119><c> scale</c><00:44:41.119><c> so</c><00:44:41.440><c> it's</c><00:44:41.680><c> not</c><00:44:42.119><c> like</c><00:44:42.319><c> as</c><00:44:42.440><c> if</c><00:44:42.599><c> it</c><00:44:42.800><c> had</c>"
+    },
+    {
+      "start": 2683.069,
+      "duration": 0.0,
+      "text": "log scale so it's not like as if it had"
+    },
+    {
+      "start": 2683.079,
+      "duration": 0.0,
+      "text": "log scale so it's not like as if it had to<00:44:43.319><c> go</c><00:44:43.839><c> it</c><00:44:44.040><c> had</c><00:44:44.200><c> to</c><00:44:44.359><c> Plateau</c><00:44:44.839><c> like</c>"
+    },
+    {
+      "start": 2684.99,
+      "duration": 0.0,
+      "text": "to go it had to Plateau like"
+    },
+    {
+      "start": 2685.0,
+      "duration": 0.0,
+      "text": "to go it had to Plateau like mathematically<00:44:45.720><c> it</c><00:44:45.839><c> could</c><00:44:46.079><c> continue</c>"
+    },
+    {
+      "start": 2686.47,
+      "duration": 0.0,
+      "text": "mathematically it could continue"
+    },
+    {
+      "start": 2686.48,
+      "duration": 0.0,
+      "text": "mathematically it could continue decreasing<00:44:47.000><c> like</c><00:44:47.200><c> this</c><00:44:47.960><c> I</c><00:44:48.000><c> mean</c><00:44:48.200><c> most</c><00:44:48.400><c> people</c>"
+    },
+    {
+      "start": 2688.589,
+      "duration": 0.0,
+      "text": "decreasing like this I mean most people"
+    },
+    {
+      "start": 2688.599,
+      "duration": 0.0,
+      "text": "decreasing like this I mean most people think<00:44:48.760><c> that</c><00:44:48.880><c> it</c><00:44:48.960><c> will</c><00:44:49.119><c> probably</c><00:44:49.359><c> Plateau</c><00:44:49.720><c> at</c>"
+    },
+    {
+      "start": 2689.829,
+      "duration": 0.0,
+      "text": "think that it will probably Plateau at"
+    },
+    {
+      "start": 2689.839,
+      "duration": 0.0,
+      "text": "think that it will probably Plateau at some<00:44:50.000><c> point</c><00:44:50.640><c> we</c><00:44:50.720><c> don't</c><00:44:50.880><c> know</c>"
+    },
+    {
+      "start": 2691.95,
+      "duration": 0.0,
+      "text": "some point we don't know"
+    },
+    {
+      "start": 2691.96,
+      "duration": 0.0,
+      "text": "some point we don't know when<00:44:53.400><c> um</c><00:44:54.400><c> okay</c><00:44:54.720><c> so</c><00:44:54.920><c> that's</c><00:44:55.440><c> I'll</c><00:44:55.680><c> talk</c><00:44:55.839><c> more</c>"
+    },
+    {
+      "start": 2695.95,
+      "duration": 0.0,
+      "text": "when um okay so that's I'll talk more"
+    },
+    {
+      "start": 2695.96,
+      "duration": 0.0,
+      "text": "when um okay so that's I'll talk more about<00:44:56.119><c> scaling</c><00:44:56.400><c> laws</c><00:44:56.720><c> now</c>"
+    },
+    {
+      "start": 2697.549,
+      "duration": 0.0,
+      "text": "about scaling laws now"
+    },
+    {
+      "start": 2697.559,
+      "duration": 0.0,
+      "text": "about scaling laws now so<00:44:58.079><c> why</c><00:44:58.280><c> are</c><00:44:58.440><c> scaling</c><00:44:58.760><c> laws</c><00:44:59.160><c> really</c><00:44:59.400><c> cool</c>"
+    },
+    {
+      "start": 2700.309,
+      "duration": 0.0,
+      "text": "so why are scaling laws really cool"
+    },
+    {
+      "start": 2700.319,
+      "duration": 0.0,
+      "text": "so why are scaling laws really cool imagine<00:45:00.640><c> that</c><00:45:00.839><c> I</c><00:45:01.000><c> give</c><00:45:01.200><c> you</c><00:45:02.040><c> um</c><00:45:02.280><c> you're</c><00:45:02.520><c> very</c>"
+    },
+    {
+      "start": 2702.71,
+      "duration": 0.0,
+      "text": "imagine that I give you um you're very"
+    },
+    {
+      "start": 2702.72,
+      "duration": 0.0,
+      "text": "imagine that I give you um you're very fortunate<00:45:03.160><c> I</c><00:45:03.280><c> gave</c><00:45:03.400><c> you</c><00:45:03.520><c> 10,000</c><00:45:04.000><c> gpus</c><00:45:04.480><c> for</c>"
+    },
+    {
+      "start": 2704.63,
+      "duration": 0.0,
+      "text": "fortunate I gave you 10,000 gpus for"
+    },
+    {
+      "start": 2704.64,
+      "duration": 0.0,
+      "text": "fortunate I gave you 10,000 gpus for this<00:45:04.920><c> month</c><00:45:05.920><c> what</c><00:45:06.160><c> model</c><00:45:06.480><c> will</c><00:45:06.599><c> you</c><00:45:06.760><c> train</c><00:45:07.400><c> how</c>"
+    },
+    {
+      "start": 2707.51,
+      "duration": 0.0,
+      "text": "this month what model will you train how"
+    },
+    {
+      "start": 2707.52,
+      "duration": 0.0,
+      "text": "this month what model will you train how do<00:45:07.640><c> you</c><00:45:07.760><c> even</c><00:45:08.000><c> go</c><00:45:08.160><c> about</c><00:45:08.400><c> answering</c><00:45:08.760><c> that</c>"
+    },
+    {
+      "start": 2708.95,
+      "duration": 0.0,
+      "text": "do you even go about answering that"
+    },
+    {
+      "start": 2708.96,
+      "duration": 0.0,
+      "text": "do you even go about answering that question<00:45:09.800><c> and</c><00:45:10.079><c> I</c><00:45:10.160><c> mean</c><00:45:10.880><c> this</c><00:45:11.000><c> is</c><00:45:11.280><c> a</c><00:45:11.599><c> a</c>"
+    },
+    {
+      "start": 2711.75,
+      "duration": 0.0,
+      "text": "question and I mean this is a a"
+    },
+    {
+      "start": 2711.76,
+      "duration": 0.0,
+      "text": "question and I mean this is a a hypothetical<00:45:12.440><c> but</c><00:45:12.559><c> that's</c><00:45:12.720><c> exactly</c><00:45:13.119><c> what</c>"
+    },
+    {
+      "start": 2713.27,
+      "duration": 0.0,
+      "text": "hypothetical but that's exactly what"
+    },
+    {
+      "start": 2713.28,
+      "duration": 0.0,
+      "text": "hypothetical but that's exactly what these<00:45:13.440><c> companies</c><00:45:14.000><c> are</c><00:45:14.240><c> faced</c><00:45:15.079><c> with</c><00:45:16.079><c> uh</c><00:45:16.240><c> the</c>"
+    },
+    {
+      "start": 2716.349,
+      "duration": 0.0,
+      "text": "these companies are faced with uh the"
+    },
+    {
+      "start": 2716.359,
+      "duration": 0.0,
+      "text": "these companies are faced with uh the old<00:45:16.839><c> pipeline</c><00:45:17.839><c> um</c><00:45:18.319><c> which</c><00:45:18.880><c> was</c><00:45:19.280><c> basically</c><00:45:19.599><c> you</c>"
+    },
+    {
+      "start": 2719.71,
+      "duration": 0.0,
+      "text": "old pipeline um which was basically you"
+    },
+    {
+      "start": 2719.72,
+      "duration": 0.0,
+      "text": "old pipeline um which was basically you tune<00:45:19.960><c> High</c><00:45:20.160><c> parameters</c><00:45:20.559><c> on</c><00:45:20.680><c> the</c><00:45:20.760><c> big</c><00:45:21.000><c> models</c>"
+    },
+    {
+      "start": 2721.71,
+      "duration": 0.0,
+      "text": "tune High parameters on the big models"
+    },
+    {
+      "start": 2721.72,
+      "duration": 0.0,
+      "text": "tune High parameters on the big models so<00:45:22.000><c> let's</c><00:45:22.160><c> say</c><00:45:22.319><c> I</c><00:45:22.480><c> have</c><00:45:22.880><c> 30</c><00:45:23.240><c> days</c><00:45:23.760><c> I</c><00:45:23.839><c> will</c><00:45:24.040><c> train</c>"
+    },
+    {
+      "start": 2724.39,
+      "duration": 0.0,
+      "text": "so let's say I have 30 days I will train"
+    },
+    {
+      "start": 2724.4,
+      "duration": 0.0,
+      "text": "so let's say I have 30 days I will train 30<00:45:24.720><c> models</c><00:45:25.040><c> for</c><00:45:25.240><c> one</c><00:45:25.480><c> day</c><00:45:26.119><c> each</c><00:45:27.000><c> I</c><00:45:27.040><c> will</c><00:45:27.200><c> pick</c>"
+    },
+    {
+      "start": 2727.39,
+      "duration": 0.0,
+      "text": "30 models for one day each I will pick"
+    },
+    {
+      "start": 2727.4,
+      "duration": 0.0,
+      "text": "30 models for one day each I will pick the<00:45:27.520><c> best</c><00:45:27.760><c> one</c><00:45:28.640><c> uh</c><00:45:28.760><c> and</c><00:45:28.920><c> that</c><00:45:29.000><c> will</c><00:45:29.160><c> be</c><00:45:29.319><c> the</c>"
+    },
+    {
+      "start": 2729.43,
+      "duration": 0.0,
+      "text": "the best one uh and that will be the"
+    },
+    {
+      "start": 2729.44,
+      "duration": 0.0,
+      "text": "the best one uh and that will be the final<00:45:29.760><c> model</c><00:45:30.079><c> that</c><00:45:30.200><c> I</c><00:45:30.280><c> will</c><00:45:30.480><c> use</c><00:45:30.680><c> in</c>"
+    },
+    {
+      "start": 2730.99,
+      "duration": 0.0,
+      "text": "final model that I will use in"
+    },
+    {
+      "start": 2731.0,
+      "duration": 0.0,
+      "text": "final model that I will use in production<00:45:32.000><c> um</c><00:45:32.280><c> that</c><00:45:32.400><c> means</c><00:45:32.640><c> that</c><00:45:32.800><c> the</c><00:45:32.920><c> model</c>"
+    },
+    {
+      "start": 2733.19,
+      "duration": 0.0,
+      "text": "production um that means that the model"
+    },
+    {
+      "start": 2733.2,
+      "duration": 0.0,
+      "text": "production um that means that the model that<00:45:33.280><c> I</c><00:45:33.440><c> actually</c><00:45:33.680><c> used</c><00:45:34.160><c> was</c><00:45:34.319><c> only</c><00:45:34.520><c> trained</c>"
+    },
+    {
+      "start": 2734.95,
+      "duration": 0.0,
+      "text": "that I actually used was only trained"
+    },
+    {
+      "start": 2734.96,
+      "duration": 0.0,
+      "text": "that I actually used was only trained for<00:45:35.119><c> one</c><00:45:35.720><c> day</c><00:45:36.720><c> the</c><00:45:36.880><c> new</c><00:45:37.119><c> pipeline</c><00:45:38.119><c> is</c><00:45:38.240><c> that</c><00:45:38.400><c> you</c>"
+    },
+    {
+      "start": 2738.549,
+      "duration": 0.0,
+      "text": "for one day the new pipeline is that you"
+    },
+    {
+      "start": 2738.559,
+      "duration": 0.0,
+      "text": "for one day the new pipeline is that you first<00:45:38.880><c> find</c><00:45:39.079><c> a</c><00:45:39.240><c> scaling</c><00:45:39.720><c> recipe</c><00:45:40.319><c> so</c><00:45:40.480><c> you</c><00:45:40.680><c> find</c>"
+    },
+    {
+      "start": 2740.95,
+      "duration": 0.0,
+      "text": "first find a scaling recipe so you find"
+    },
+    {
+      "start": 2740.96,
+      "duration": 0.0,
+      "text": "first find a scaling recipe so you find something<00:45:41.400><c> that</c><00:45:41.559><c> tells</c><00:45:41.839><c> you</c><00:45:42.079><c> for</c><00:45:42.280><c> example</c><00:45:43.160><c> oh</c>"
+    },
+    {
+      "start": 2743.549,
+      "duration": 0.0,
+      "text": "something that tells you for example oh"
+    },
+    {
+      "start": 2743.559,
+      "duration": 0.0,
+      "text": "something that tells you for example oh like<00:45:43.680><c> one</c><00:45:43.880><c> common</c><00:45:44.119><c> thing</c><00:45:44.280><c> is</c><00:45:44.400><c> that</c><00:45:44.559><c> if</c><00:45:44.640><c> you</c>"
+    },
+    {
+      "start": 2744.79,
+      "duration": 0.0,
+      "text": "like one common thing is that if you"
+    },
+    {
+      "start": 2744.8,
+      "duration": 0.0,
+      "text": "like one common thing is that if you increase<00:45:45.160><c> the</c><00:45:45.280><c> size</c><00:45:45.480><c> of</c><00:45:45.559><c> your</c><00:45:45.680><c> model</c><00:45:45.960><c> you</c>"
+    },
+    {
+      "start": 2745.99,
+      "duration": 0.0,
+      "text": "increase the size of your model you"
+    },
+    {
+      "start": 2746.0,
+      "duration": 0.0,
+      "text": "increase the size of your model you should<00:45:46.160><c> decrease</c><00:45:46.440><c> your</c><00:45:46.559><c> learning</c><00:45:46.839><c> rate</c><00:45:47.319><c> so</c>"
+    },
+    {
+      "start": 2747.43,
+      "duration": 0.0,
+      "text": "should decrease your learning rate so"
+    },
+    {
+      "start": 2747.44,
+      "duration": 0.0,
+      "text": "should decrease your learning rate so you<00:45:47.559><c> find</c><00:45:47.720><c> a</c><00:45:47.839><c> scaling</c><00:45:48.240><c> recipe</c><00:45:48.800><c> such</c><00:45:49.040><c> that</c><00:45:49.200><c> you</c>"
+    },
+    {
+      "start": 2749.309,
+      "duration": 0.0,
+      "text": "you find a scaling recipe such that you"
+    },
+    {
+      "start": 2749.319,
+      "duration": 0.0,
+      "text": "you find a scaling recipe such that you know<00:45:49.720><c> if</c><00:45:49.880><c> I</c><00:45:50.040><c> increase</c><00:45:50.400><c> the</c><00:45:50.839><c> the</c><00:45:51.280><c> the</c><00:45:51.680><c> the</c><00:45:51.839><c> size</c>"
+    },
+    {
+      "start": 2751.99,
+      "duration": 0.0,
+      "text": "know if I increase the the the the size"
+    },
+    {
+      "start": 2752.0,
+      "duration": 0.0,
+      "text": "know if I increase the the the the size of<00:45:52.160><c> my</c><00:45:52.280><c> model</c><00:45:52.640><c> here's</c><00:45:52.839><c> what</c><00:45:52.960><c> I</c><00:45:53.040><c> should</c><00:45:53.240><c> do</c><00:45:53.440><c> with</c>"
+    },
+    {
+      "start": 2753.549,
+      "duration": 0.0,
+      "text": "of my model here's what I should do with"
+    },
+    {
+      "start": 2753.559,
+      "duration": 0.0,
+      "text": "of my model here's what I should do with some<00:45:53.760><c> high</c><00:45:54.319><c> parameters</c><00:45:55.319><c> then</c><00:45:55.440><c> you</c><00:45:55.839><c> tune</c><00:45:56.160><c> your</c>"
+    },
+    {
+      "start": 2756.309,
+      "duration": 0.0,
+      "text": "some high parameters then you tune your"
+    },
+    {
+      "start": 2756.319,
+      "duration": 0.0,
+      "text": "some high parameters then you tune your high<00:45:56.559><c> parameter</c>"
+    },
+    {
+      "start": 2757.91,
+      "duration": 0.0,
+      "text": "high parameter"
+    },
+    {
+      "start": 2757.92,
+      "duration": 0.0,
+      "text": "high parameter on<00:45:58.280><c> smaller</c><00:45:58.720><c> models</c><00:45:59.400><c> of</c><00:45:59.640><c> different</c><00:45:59.920><c> sizes</c>"
+    },
+    {
+      "start": 2760.67,
+      "duration": 0.0,
+      "text": "on smaller models of different sizes"
+    },
+    {
+      "start": 2760.68,
+      "duration": 0.0,
+      "text": "on smaller models of different sizes let's<00:46:00.880><c> say</c><00:46:01.160><c> I</c><00:46:01.240><c> will</c><00:46:01.440><c> say</c><00:46:01.680><c> for</c><00:46:01.920><c> 3</c><00:46:02.160><c> Days</c><00:46:02.400><c> of</c><00:46:02.559><c> my</c><00:46:02.720><c> 30</c>"
+    },
+    {
+      "start": 2763.03,
+      "duration": 0.0,
+      "text": "let's say I will say for 3 Days of my 30"
+    },
+    {
+      "start": 2763.04,
+      "duration": 0.0,
+      "text": "let's say I will say for 3 Days of my 30 days<00:46:03.559><c> I</c><00:46:03.640><c> will</c><00:46:03.960><c> train</c><00:46:04.440><c> many</c><00:46:04.680><c> different</c><00:46:05.000><c> models</c>"
+    },
+    {
+      "start": 2765.309,
+      "duration": 0.0,
+      "text": "days I will train many different models"
+    },
+    {
+      "start": 2765.319,
+      "duration": 0.0,
+      "text": "days I will train many different models and<00:46:05.440><c> I</c><00:46:05.480><c> would</c><00:46:05.640><c> do</c><00:46:05.839><c> highper</c><00:46:06.160><c> parameter</c><00:46:06.520><c> tuning</c>"
+    },
+    {
+      "start": 2767.069,
+      "duration": 0.0,
+      "text": "and I would do highper parameter tuning"
+    },
+    {
+      "start": 2767.079,
+      "duration": 0.0,
+      "text": "and I would do highper parameter tuning on<00:46:07.240><c> these</c><00:46:07.400><c> small</c><00:46:07.680><c> models</c><00:46:08.079><c> each</c><00:46:08.200><c> of</c><00:46:08.400><c> different</c>"
+    },
+    {
+      "start": 2768.67,
+      "duration": 0.0,
+      "text": "on these small models each of different"
+    },
+    {
+      "start": 2768.68,
+      "duration": 0.0,
+      "text": "on these small models each of different sizes<00:46:09.520><c> then</c><00:46:09.640><c> I</c><00:46:09.760><c> will</c><00:46:09.960><c> fit</c><00:46:10.240><c> a</c><00:46:10.400><c> scaling</c><00:46:10.760><c> law</c><00:46:11.440><c> and</c>"
+    },
+    {
+      "start": 2771.63,
+      "duration": 0.0,
+      "text": "sizes then I will fit a scaling law and"
+    },
+    {
+      "start": 2771.64,
+      "duration": 0.0,
+      "text": "sizes then I will fit a scaling law and try<00:46:11.839><c> to</c><00:46:12.440><c> extrapolate</c><00:46:13.440><c> from</c><00:46:13.720><c> these</c><00:46:13.880><c> smaller</c>"
+    },
+    {
+      "start": 2774.309,
+      "duration": 0.0,
+      "text": "try to extrapolate from these smaller"
+    },
+    {
+      "start": 2774.319,
+      "duration": 0.0,
+      "text": "try to extrapolate from these smaller models<00:46:15.319><c> which</c><00:46:15.559><c> one</c><00:46:15.800><c> will</c><00:46:16.000><c> be</c><00:46:16.200><c> the</c><00:46:16.400><c> best</c><00:46:17.280><c> if</c><00:46:17.480><c> I</c>"
+    },
+    {
+      "start": 2777.71,
+      "duration": 0.0,
+      "text": "models which one will be the best if I"
+    },
+    {
+      "start": 2777.72,
+      "duration": 0.0,
+      "text": "models which one will be the best if I if<00:46:17.800><c> I</c><00:46:17.920><c> train</c><00:46:18.119><c> it</c><00:46:18.240><c> for</c><00:46:18.440><c> much</c><00:46:18.920><c> longer</c><00:46:19.920><c> or</c><00:46:20.040><c> sorry</c>"
+    },
+    {
+      "start": 2780.39,
+      "duration": 0.0,
+      "text": "if I train it for much longer or sorry"
+    },
+    {
+      "start": 2780.4,
+      "duration": 0.0,
+      "text": "if I train it for much longer or sorry if<00:46:20.559><c> I</c><00:46:20.760><c> train</c><00:46:21.040><c> it</c><00:46:21.200><c> for</c><00:46:21.359><c> a</c><00:46:21.520><c> larger</c><00:46:22.079><c> model</c><00:46:23.079><c> and</c>"
+    },
+    {
+      "start": 2783.19,
+      "duration": 0.0,
+      "text": "if I train it for a larger model and"
+    },
+    {
+      "start": 2783.2,
+      "duration": 0.0,
+      "text": "if I train it for a larger model and then<00:46:23.359><c> I</c><00:46:23.440><c> will</c><00:46:23.559><c> train</c><00:46:23.800><c> the</c><00:46:23.920><c> final</c><00:46:24.240><c> huge</c><00:46:24.520><c> model</c>"
+    },
+    {
+      "start": 2784.87,
+      "duration": 0.0,
+      "text": "then I will train the final huge model"
+    },
+    {
+      "start": 2784.88,
+      "duration": 0.0,
+      "text": "then I will train the final huge model for<00:46:25.119><c> 27</c><00:46:25.640><c> days</c><00:46:25.920><c> instead</c><00:46:26.200><c> of</c><00:46:26.359><c> just</c><00:46:26.480><c> one</c><00:46:26.720><c> day</c>"
+    },
+    {
+      "start": 2787.79,
+      "duration": 0.0,
+      "text": "for 27 days instead of just one day"
+    },
+    {
+      "start": 2787.8,
+      "duration": 0.0,
+      "text": "for 27 days instead of just one day um<00:46:28.319><c> so</c><00:46:28.520><c> the</c><00:46:28.680><c> new</c><00:46:28.920><c> pipeline</c><00:46:29.680><c> is</c><00:46:29.960><c> not</c><00:46:30.599><c> train</c>"
+    },
+    {
+      "start": 2791.109,
+      "duration": 0.0,
+      "text": "um so the new pipeline is not train"
+    },
+    {
+      "start": 2791.119,
+      "duration": 0.0,
+      "text": "um so the new pipeline is not train things<00:46:31.559><c> or</c><00:46:31.760><c> do</c><00:46:31.960><c> high</c><00:46:32.160><c> prity</c><00:46:32.520><c> tuning</c><00:46:32.839><c> on</c><00:46:33.000><c> the</c>"
+    },
+    {
+      "start": 2793.19,
+      "duration": 0.0,
+      "text": "things or do high prity tuning on the"
+    },
+    {
+      "start": 2793.2,
+      "duration": 0.0,
+      "text": "things or do high prity tuning on the real<00:46:33.520><c> scale</c><00:46:33.800><c> of</c><00:46:33.920><c> the</c><00:46:34.000><c> model</c><00:46:34.240><c> that</c><00:46:34.319><c> you're</c>"
+    },
+    {
+      "start": 2794.43,
+      "duration": 0.0,
+      "text": "real scale of the model that you're"
+    },
+    {
+      "start": 2794.44,
+      "duration": 0.0,
+      "text": "real scale of the model that you're going<00:46:34.520><c> to</c><00:46:34.599><c> use</c><00:46:34.720><c> in</c><00:46:34.960><c> practice</c><00:46:35.599><c> but</c><00:46:35.760><c> do</c><00:46:35.960><c> things</c>"
+    },
+    {
+      "start": 2796.109,
+      "duration": 0.0,
+      "text": "going to use in practice but do things"
+    },
+    {
+      "start": 2796.119,
+      "duration": 0.0,
+      "text": "going to use in practice but do things on<00:46:36.319><c> smaller</c><00:46:37.160><c> ones</c><00:46:38.160><c> at</c><00:46:38.359><c> different</c><00:46:38.640><c> scales</c><00:46:39.480><c> try</c>"
+    },
+    {
+      "start": 2799.75,
+      "duration": 0.0,
+      "text": "on smaller ones at different scales try"
+    },
+    {
+      "start": 2799.76,
+      "duration": 0.0,
+      "text": "on smaller ones at different scales try to<00:46:40.000><c> predict</c><00:46:40.559><c> how</c><00:46:40.720><c> well</c><00:46:40.920><c> they</c><00:46:41.040><c> will</c><00:46:41.240><c> perform</c>"
+    },
+    {
+      "start": 2801.589,
+      "duration": 0.0,
+      "text": "to predict how well they will perform"
+    },
+    {
+      "start": 2801.599,
+      "duration": 0.0,
+      "text": "to predict how well they will perform once<00:46:41.720><c> you</c><00:46:41.880><c> make</c><00:46:42.079><c> them</c><00:46:42.240><c> bigger</c><00:46:43.040><c> I</c><00:46:43.119><c> will</c><00:46:43.359><c> give</c><00:46:43.720><c> I</c>"
+    },
+    {
+      "start": 2803.75,
+      "duration": 0.0,
+      "text": "once you make them bigger I will give I"
+    },
+    {
+      "start": 2803.76,
+      "duration": 0.0,
+      "text": "once you make them bigger I will give I will<00:46:43.960><c> give</c><00:46:44.079><c> you</c><00:46:44.240><c> a</c><00:46:44.400><c> very</c><00:46:44.599><c> concrete</c><00:46:45.000><c> example</c>"
+    },
+    {
+      "start": 2805.43,
+      "duration": 0.0,
+      "text": "will give you a very concrete example"
+    },
+    {
+      "start": 2805.44,
+      "duration": 0.0,
+      "text": "will give you a very concrete example right<00:46:45.599><c> now</c><00:46:46.440><c> uh</c><00:46:46.599><c> let's</c><00:46:46.839><c> say</c><00:46:47.319><c> Transformers</c>"
+    },
+    {
+      "start": 2808.069,
+      "duration": 0.0,
+      "text": "right now uh let's say Transformers"
+    },
+    {
+      "start": 2808.079,
+      "duration": 0.0,
+      "text": "right now uh let's say Transformers versus<00:46:48.680><c> lstms</c><00:46:49.680><c> let's</c><00:46:49.839><c> say</c><00:46:50.040><c> you</c><00:46:50.480><c> you</c><00:46:50.640><c> have</c>"
+    },
+    {
+      "start": 2810.79,
+      "duration": 0.0,
+      "text": "versus lstms let's say you you have"
+    },
+    {
+      "start": 2810.8,
+      "duration": 0.0,
+      "text": "versus lstms let's say you you have these<00:46:50.960><c> 10,000</c><00:46:51.400><c> gpus</c><00:46:51.880><c> you</c><00:46:52.000><c> will</c><00:46:52.200><c> not</c><00:46:52.319><c> sure</c>"
+    },
+    {
+      "start": 2812.589,
+      "duration": 0.0,
+      "text": "these 10,000 gpus you will not sure"
+    },
+    {
+      "start": 2812.599,
+      "duration": 0.0,
+      "text": "these 10,000 gpus you will not sure which<00:46:52.720><c> one</c><00:46:52.880><c> you</c><00:46:52.960><c> should</c><00:46:53.119><c> be</c><00:46:53.240><c> using</c><00:46:53.559><c> should</c><00:46:53.720><c> I</c>"
+    },
+    {
+      "start": 2813.829,
+      "duration": 0.0,
+      "text": "which one you should be using should I"
+    },
+    {
+      "start": 2813.839,
+      "duration": 0.0,
+      "text": "which one you should be using should I be<00:46:53.960><c> using</c><00:46:54.280><c> Transformer</c><00:46:54.800><c> based</c><00:46:55.000><c> model</c><00:46:55.240><c> or</c><00:46:55.400><c> LCM</c>"
+    },
+    {
+      "start": 2815.87,
+      "duration": 0.0,
+      "text": "be using Transformer based model or LCM"
+    },
+    {
+      "start": 2815.88,
+      "duration": 0.0,
+      "text": "be using Transformer based model or LCM based<00:46:56.119><c> model</c><00:46:56.680><c> what</c><00:46:56.960><c> I</c><00:46:57.040><c> will</c><00:46:57.160><c> do</c><00:46:57.280><c> is</c><00:46:57.400><c> I</c><00:46:57.480><c> will</c>"
+    },
+    {
+      "start": 2817.589,
+      "duration": 0.0,
+      "text": "based model what I will do is I will"
+    },
+    {
+      "start": 2817.599,
+      "duration": 0.0,
+      "text": "based model what I will do is I will train<00:46:57.920><c> Transformers</c><00:46:58.920><c> at</c><00:46:59.119><c> different</c><00:46:59.359><c> skills</c>"
+    },
+    {
+      "start": 2819.99,
+      "duration": 0.0,
+      "text": "train Transformers at different skills"
+    },
+    {
+      "start": 2820.0,
+      "duration": 0.0,
+      "text": "train Transformers at different skills so<00:47:00.280><c> here</c><00:47:00.400><c> you</c><00:47:00.520><c> see</c><00:47:00.760><c> different</c><00:47:01.040><c> parameters</c><00:47:01.440><c> on</c>"
+    },
+    {
+      "start": 2821.589,
+      "duration": 0.0,
+      "text": "so here you see different parameters on"
+    },
+    {
+      "start": 2821.599,
+      "duration": 0.0,
+      "text": "so here you see different parameters on the<00:47:01.839><c> x-axis</c><00:47:02.760><c> Y</c><00:47:02.920><c> axis</c><00:47:03.200><c> is</c><00:47:03.359><c> my</c><00:47:03.520><c> test</c><00:47:03.760><c> loss</c><00:47:04.400><c> I</c><00:47:04.480><c> will</c>"
+    },
+    {
+      "start": 2824.63,
+      "duration": 0.0,
+      "text": "the x-axis Y axis is my test loss I will"
+    },
+    {
+      "start": 2824.64,
+      "duration": 0.0,
+      "text": "the x-axis Y axis is my test loss I will then<00:47:04.880><c> train</c><00:47:05.280><c> different</c><00:47:05.920><c> different</c><00:47:06.160><c> lstms</c><00:47:07.160><c> at</c>"
+    },
+    {
+      "start": 2827.309,
+      "duration": 0.0,
+      "text": "then train different different lstms at"
+    },
+    {
+      "start": 2827.319,
+      "duration": 0.0,
+      "text": "then train different different lstms at different<00:47:07.559><c> scales</c><00:47:08.559><c> once</c><00:47:08.760><c> I</c><00:47:08.920><c> have</c><00:47:09.079><c> these</c>"
+    },
+    {
+      "start": 2829.27,
+      "duration": 0.0,
+      "text": "different scales once I have these"
+    },
+    {
+      "start": 2829.28,
+      "duration": 0.0,
+      "text": "different scales once I have these points<00:47:09.920><c> I</c><00:47:10.000><c> will</c><00:47:10.200><c> see</c><00:47:10.559><c> oh</c><00:47:10.720><c> it</c><00:47:10.800><c> kind</c><00:47:10.920><c> of</c><00:47:11.079><c> fits</c><00:47:11.319><c> a</c>"
+    },
+    {
+      "start": 2831.67,
+      "duration": 0.0,
+      "text": "points I will see oh it kind of fits a"
+    },
+    {
+      "start": 2831.68,
+      "duration": 0.0,
+      "text": "points I will see oh it kind of fits a scaling<00:47:12.119><c> law</c><00:47:12.559><c> I</c><00:47:12.640><c> will</c><00:47:12.880><c> fit</c><00:47:13.040><c> my</c><00:47:13.119><c> scaling</c><00:47:13.440><c> law</c>"
+    },
+    {
+      "start": 2833.91,
+      "duration": 0.0,
+      "text": "scaling law I will fit my scaling law"
+    },
+    {
+      "start": 2833.92,
+      "duration": 0.0,
+      "text": "scaling law I will fit my scaling law and<00:47:14.040><c> then</c><00:47:14.160><c> I</c><00:47:14.240><c> will</c><00:47:14.400><c> be</c><00:47:14.520><c> able</c><00:47:14.680><c> to</c><00:47:15.119><c> predict</c><00:47:16.119><c> oh</c><00:47:16.480><c> if</c>"
+    },
+    {
+      "start": 2836.63,
+      "duration": 0.0,
+      "text": "and then I will be able to predict oh if"
+    },
+    {
+      "start": 2836.64,
+      "duration": 0.0,
+      "text": "and then I will be able to predict oh if I<00:47:16.839><c> had</c><00:47:17.559><c> 10</c><00:47:17.839><c> times</c><00:47:18.079><c> more</c><00:47:18.280><c> compute</c><00:47:18.800><c> here's</c><00:47:19.040><c> how</c>"
+    },
+    {
+      "start": 2839.19,
+      "duration": 0.0,
+      "text": "I had 10 times more compute here's how"
+    },
+    {
+      "start": 2839.2,
+      "duration": 0.0,
+      "text": "I had 10 times more compute here's how well<00:47:19.319><c> I</c><00:47:19.400><c> would</c><00:47:19.640><c> perform</c><00:47:20.119><c> for</c><00:47:20.359><c> the</c><00:47:20.520><c> LM</c><00:47:21.359><c> it's</c>"
+    },
+    {
+      "start": 2841.549,
+      "duration": 0.0,
+      "text": "well I would perform for the LM it's"
+    },
+    {
+      "start": 2841.559,
+      "duration": 0.0,
+      "text": "well I would perform for the LM it's actually<00:47:21.760><c> slightly</c><00:47:22.079><c> less</c><00:47:22.240><c> linear</c><00:47:22.559><c> for</c><00:47:22.680><c> the</c>"
+    },
+    {
+      "start": 2842.79,
+      "duration": 0.0,
+      "text": "actually slightly less linear for the"
+    },
+    {
+      "start": 2842.8,
+      "duration": 0.0,
+      "text": "actually slightly less linear for the lstm<00:47:23.599><c> but</c><00:47:23.800><c> like</c><00:47:23.920><c> you</c><00:47:24.040><c> could</c><00:47:24.319><c> probably</c><00:47:24.599><c> try</c><00:47:24.800><c> to</c>"
+    },
+    {
+      "start": 2844.99,
+      "duration": 0.0,
+      "text": "lstm but like you could probably try to"
+    },
+    {
+      "start": 2845.0,
+      "duration": 0.0,
+      "text": "lstm but like you could probably try to predict<00:47:25.520><c> where</c><00:47:25.720><c> you</c><00:47:25.800><c> would</c><00:47:26.000><c> end</c><00:47:26.240><c> up</c><00:47:26.640><c> and</c>"
+    },
+    {
+      "start": 2846.91,
+      "duration": 0.0,
+      "text": "predict where you would end up and"
+    },
+    {
+      "start": 2846.92,
+      "duration": 0.0,
+      "text": "predict where you would end up and clearly<00:47:27.200><c> from</c><00:47:27.400><c> this</c><00:47:27.559><c> plot</c><00:47:28.200><c> you</c><00:47:28.319><c> would</c><00:47:28.440><c> see</c>"
+    },
+    {
+      "start": 2848.589,
+      "duration": 0.0,
+      "text": "clearly from this plot you would see"
+    },
+    {
+      "start": 2848.599,
+      "duration": 0.0,
+      "text": "clearly from this plot you would see that<00:47:28.720><c> Transformers</c><00:47:29.200><c> are</c><00:47:29.599><c> better</c><00:47:30.599><c> um</c><00:47:30.920><c> one</c>"
+    },
+    {
+      "start": 2851.109,
+      "duration": 0.0,
+      "text": "that Transformers are better um one"
+    },
+    {
+      "start": 2851.119,
+      "duration": 0.0,
+      "text": "that Transformers are better um one thing<00:47:31.240><c> to</c><00:47:31.400><c> notice</c><00:47:31.720><c> when</c><00:47:31.839><c> you</c><00:47:31.960><c> read</c><00:47:32.240><c> these</c><00:47:32.440><c> type</c>"
+    },
+    {
+      "start": 2852.589,
+      "duration": 0.0,
+      "text": "thing to notice when you read these type"
+    },
+    {
+      "start": 2852.599,
+      "duration": 0.0,
+      "text": "thing to notice when you read these type of<00:47:32.760><c> scaling</c><00:47:33.079><c> laws</c><00:47:33.359><c> is</c><00:47:33.480><c> that</c><00:47:33.640><c> are</c><00:47:33.800><c> two</c><00:47:33.960><c> things</c>"
+    },
+    {
+      "start": 2854.109,
+      "duration": 0.0,
+      "text": "of scaling laws is that are two things"
+    },
+    {
+      "start": 2854.119,
+      "duration": 0.0,
+      "text": "of scaling laws is that are two things that<00:47:34.240><c> are</c><00:47:34.599><c> important</c><00:47:35.599><c> uh</c><00:47:36.000><c> one</c><00:47:36.880><c> is</c><00:47:37.680><c> really</c><00:47:38.000><c> your</c>"
+    },
+    {
+      "start": 2858.19,
+      "duration": 0.0,
+      "text": "that are important uh one is really your"
+    },
+    {
+      "start": 2858.2,
+      "duration": 0.0,
+      "text": "that are important uh one is really your scaling<00:47:38.960><c> rate</c><00:47:39.960><c> uh</c><00:47:40.119><c> which</c><00:47:40.280><c> is</c><00:47:40.480><c> kind</c><00:47:40.640><c> of</c><00:47:41.119><c> the</c><00:47:42.119><c> uh</c>"
+    },
+    {
+      "start": 2862.27,
+      "duration": 0.0,
+      "text": "scaling rate uh which is kind of the uh"
+    },
+    {
+      "start": 2862.28,
+      "duration": 0.0,
+      "text": "scaling rate uh which is kind of the uh the<00:47:42.480><c> slope</c><00:47:43.480><c> of</c><00:47:43.800><c> the</c><00:47:44.319><c> the</c><00:47:44.440><c> slope</c><00:47:44.800><c> of</c><00:47:44.920><c> the</c>"
+    },
+    {
+      "start": 2865.03,
+      "duration": 0.0,
+      "text": "the slope of the the slope of the"
+    },
+    {
+      "start": 2865.04,
+      "duration": 0.0,
+      "text": "the slope of the the slope of the scaling<00:47:45.359><c> law</c><00:47:45.800><c> the</c><00:47:45.960><c> other</c><00:47:46.160><c> thing</c><00:47:46.400><c> is</c><00:47:46.800><c> your</c><00:47:47.800><c> um</c>"
+    },
+    {
+      "start": 2868.309,
+      "duration": 0.0,
+      "text": "scaling law the other thing is your um"
+    },
+    {
+      "start": 2868.319,
+      "duration": 0.0,
+      "text": "scaling law the other thing is your um your<00:47:49.040><c> intercept</c><00:47:50.040><c> like</c><00:47:50.160><c> you</c><00:47:50.280><c> could</c><00:47:50.480><c> start</c>"
+    },
+    {
+      "start": 2871.069,
+      "duration": 0.0,
+      "text": "your intercept like you could start"
+    },
+    {
+      "start": 2871.079,
+      "duration": 0.0,
+      "text": "your intercept like you could start worse<00:47:51.640><c> but</c><00:47:51.880><c> actually</c><00:47:52.119><c> become</c><00:47:52.480><c> better</c><00:47:52.839><c> over</c>"
+    },
+    {
+      "start": 2873.109,
+      "duration": 0.0,
+      "text": "worse but actually become better over"
+    },
+    {
+      "start": 2873.119,
+      "duration": 0.0,
+      "text": "worse but actually become better over time<00:47:53.640><c> it</c><00:47:53.760><c> just</c><00:47:53.920><c> happens</c><00:47:54.160><c> that</c><00:47:54.359><c> lstms</c><00:47:54.839><c> are</c>"
+    },
+    {
+      "start": 2874.99,
+      "duration": 0.0,
+      "text": "time it just happens that lstms are"
+    },
+    {
+      "start": 2875.0,
+      "duration": 0.0,
+      "text": "time it just happens that lstms are worse<00:47:55.200><c> for</c><00:47:55.400><c> both</c><00:47:56.079><c> uh</c><00:47:56.160><c> but</c><00:47:56.280><c> I</c><00:47:56.359><c> could</c><00:47:56.520><c> show</c><00:47:56.680><c> you</c>"
+    },
+    {
+      "start": 2877.03,
+      "duration": 0.0,
+      "text": "worse for both uh but I could show you"
+    },
+    {
+      "start": 2877.04,
+      "duration": 0.0,
+      "text": "worse for both uh but I could show you another<00:47:57.280><c> one</c><00:47:57.839><c> where</c><00:47:58.119><c> things</c><00:47:58.720><c> you</c><00:47:58.839><c> can</c><00:47:59.079><c> predict</c>"
+    },
+    {
+      "start": 2879.51,
+      "duration": 0.0,
+      "text": "another one where things you can predict"
+    },
+    {
+      "start": 2879.52,
+      "duration": 0.0,
+      "text": "another one where things you can predict that<00:47:59.720><c> actually</c><00:48:00.440><c> after</c><00:48:00.640><c> a</c><00:48:00.760><c> certain</c><00:48:01.079><c> scale</c>"
+    },
+    {
+      "start": 2881.349,
+      "duration": 0.0,
+      "text": "that actually after a certain scale"
+    },
+    {
+      "start": 2881.359,
+      "duration": 0.0,
+      "text": "that actually after a certain scale you're<00:48:01.559><c> better</c><00:48:01.880><c> off</c><00:48:02.280><c> using</c><00:48:02.640><c> that</c><00:48:02.800><c> type</c><00:48:02.960><c> of</c>"
+    },
+    {
+      "start": 2883.069,
+      "duration": 0.0,
+      "text": "you're better off using that type of"
+    },
+    {
+      "start": 2883.079,
+      "duration": 0.0,
+      "text": "you're better off using that type of model<00:48:03.400><c> than</c><00:48:03.599><c> others</c><00:48:04.319><c> uh</c><00:48:04.400><c> so</c><00:48:04.559><c> that's</c><00:48:04.760><c> why</c>"
+    },
+    {
+      "start": 2885.03,
+      "duration": 0.0,
+      "text": "model than others uh so that's why"
+    },
+    {
+      "start": 2885.04,
+      "duration": 0.0,
+      "text": "model than others uh so that's why scaling<00:48:05.440><c> laws</c><00:48:06.040><c> are</c><00:48:06.240><c> actually</c><00:48:06.520><c> really</c>"
+    },
+    {
+      "start": 2887.67,
+      "duration": 0.0,
+      "text": "scaling laws are actually really"
+    },
+    {
+      "start": 2887.68,
+      "duration": 0.0,
+      "text": "scaling laws are actually really useful<00:48:08.680><c> any</c><00:48:08.880><c> questions</c><00:48:09.160><c> on</c>"
+    },
+    {
+      "start": 2891.44,
+      "duration": 0.0,
+      "text": "that<00:48:12.440><c> yeah</c><00:48:13.040><c> so</c><00:48:13.640><c> these</c><00:48:13.800><c> are</c><00:48:13.960><c> all</c><00:48:14.200><c> kind</c><00:48:14.319><c> of</c><00:48:14.800><c> very</c>"
+    },
+    {
+      "start": 2895.67,
+      "duration": 0.0,
+      "text": "that yeah so these are all kind of very"
+    },
+    {
+      "start": 2895.68,
+      "duration": 0.0,
+      "text": "that yeah so these are all kind of very how<00:48:15.920><c> how</c><00:48:16.160><c> sensitive</c><00:48:16.520><c> are</c><00:48:16.720><c> these</c><00:48:16.880><c> to</c><00:48:17.079><c> like</c>"
+    },
+    {
+      "start": 2897.23,
+      "duration": 0.0,
+      "text": "how how sensitive are these to like"
+    },
+    {
+      "start": 2897.24,
+      "duration": 0.0,
+      "text": "how how sensitive are these to like small<00:48:17.559><c> differences</c><00:48:17.960><c> in</c><00:48:18.079><c> the</c><00:48:18.240><c> architecture</c>"
+    },
+    {
+      "start": 2898.95,
+      "duration": 0.0,
+      "text": "small differences in the architecture"
+    },
+    {
+      "start": 2898.96,
+      "duration": 0.0,
+      "text": "small differences in the architecture like<00:48:19.960><c> one</c><00:48:20.520><c> one</c><00:48:20.839><c> like</c><00:48:21.000><c> Transformer</c>"
+    },
+    {
+      "start": 2901.51,
+      "duration": 0.0,
+      "text": "like one one like Transformer"
+    },
+    {
+      "start": 2901.52,
+      "duration": 0.0,
+      "text": "like one one like Transformer architecture<00:48:22.000><c> versus</c><00:48:22.319><c> another</c><00:48:22.599><c> Transformer</c>"
+    },
+    {
+      "start": 2903.069,
+      "duration": 0.0,
+      "text": "architecture versus another Transformer"
+    },
+    {
+      "start": 2903.079,
+      "duration": 0.0,
+      "text": "architecture versus another Transformer architecture<00:48:23.720><c> you</c><00:48:23.920><c> basically</c><00:48:24.280><c> have</c><00:48:24.400><c> to</c><00:48:24.599><c> like</c>"
+    },
+    {
+      "start": 2905.109,
+      "duration": 0.0,
+      "text": "architecture you basically have to like"
+    },
+    {
+      "start": 2905.119,
+      "duration": 0.0,
+      "text": "architecture you basically have to like fit<00:48:25.359><c> your</c><00:48:25.559><c> own</c><00:48:25.920><c> curve</c><00:48:26.480><c> and</c><00:48:26.599><c> make</c><00:48:26.760><c> basically</c>"
+    },
+    {
+      "start": 2907.069,
+      "duration": 0.0,
+      "text": "fit your own curve and make basically"
+    },
+    {
+      "start": 2907.079,
+      "duration": 0.0,
+      "text": "fit your own curve and make basically say<00:48:27.240><c> like</c><00:48:27.359><c> oh</c><00:48:27.520><c> scaling</c><00:48:27.880><c> law</c><00:48:28.000><c> has</c><00:48:28.079><c> tell</c><00:48:28.240><c> me</c>"
+    },
+    {
+      "start": 2908.43,
+      "duration": 0.0,
+      "text": "say like oh scaling law has tell me"
+    },
+    {
+      "start": 2908.44,
+      "duration": 0.0,
+      "text": "say like oh scaling law has tell me there<00:48:28.559><c> should</c><00:48:28.800><c> be</c><00:48:29.440><c> some</c><00:48:29.760><c> like</c><00:48:29.920><c> logarithmic</c>"
+    },
+    {
+      "start": 2910.51,
+      "duration": 0.0,
+      "text": "there should be some like logarithmic"
+    },
+    {
+      "start": 2910.52,
+      "duration": 0.0,
+      "text": "there should be some like logarithmic function<00:48:31.480><c> let</c><00:48:31.640><c> me</c><00:48:32.640><c> extrapolate</c><00:48:33.240><c> that</c><00:48:33.400><c> for</c><00:48:33.599><c> my</c>"
+    },
+    {
+      "start": 2914.47,
+      "duration": 0.0,
+      "text": "function let me extrapolate that for my"
+    },
+    {
+      "start": 2914.48,
+      "duration": 0.0,
+      "text": "function let me extrapolate that for my own<00:48:35.480><c> yeah</c><00:48:35.760><c> so</c><00:48:36.599><c> uh</c><00:48:36.720><c> usually</c><00:48:37.040><c> for</c><00:48:37.200><c> example</c><00:48:37.480><c> if</c>"
+    },
+    {
+      "start": 2917.549,
+      "duration": 0.0,
+      "text": "own yeah so uh usually for example if"
+    },
+    {
+      "start": 2917.559,
+      "duration": 0.0,
+      "text": "own yeah so uh usually for example if you're<00:48:37.640><c> an</c><00:48:37.760><c> academic</c><00:48:38.200><c> and</c><00:48:38.280><c> you</c><00:48:38.400><c> want</c><00:48:38.520><c> to</c><00:48:38.800><c> now</c>"
+    },
+    {
+      "start": 2919.03,
+      "duration": 0.0,
+      "text": "you're an academic and you want to now"
+    },
+    {
+      "start": 2919.04,
+      "duration": 0.0,
+      "text": "you're an academic and you want to now at<00:48:39.119><c> least</c><00:48:39.319><c> that's</c><00:48:39.559><c> like</c><00:48:39.800><c> pretty</c><00:48:40.520><c> recent</c><00:48:41.040><c> and</c>"
+    },
+    {
+      "start": 2921.15,
+      "duration": 0.0,
+      "text": "at least that's like pretty recent and"
+    },
+    {
+      "start": 2921.16,
+      "duration": 0.0,
+      "text": "at least that's like pretty recent and you<00:48:41.240><c> want</c><00:48:41.359><c> to</c><00:48:41.559><c> propose</c><00:48:41.839><c> a</c><00:48:41.960><c> new</c><00:48:42.240><c> like</c>"
+    },
+    {
+      "start": 2922.829,
+      "duration": 0.0,
+      "text": "you want to propose a new like"
+    },
+    {
+      "start": 2922.839,
+      "duration": 0.0,
+      "text": "you want to propose a new like activation<00:48:43.839><c> uh</c><00:48:44.000><c> that's</c><00:48:44.160><c> exactly</c><00:48:44.480><c> what</c><00:48:44.559><c> you</c>"
+    },
+    {
+      "start": 2924.63,
+      "duration": 0.0,
+      "text": "activation uh that's exactly what you"
+    },
+    {
+      "start": 2924.64,
+      "duration": 0.0,
+      "text": "activation uh that's exactly what you will<00:48:44.800><c> do</c><00:48:45.000><c> you</c><00:48:45.119><c> will</c><00:48:45.359><c> fit</c><00:48:45.520><c> a</c><00:48:45.599><c> scaling</c><00:48:45.920><c> law</c><00:48:46.359><c> show</c>"
+    },
+    {
+      "start": 2926.67,
+      "duration": 0.0,
+      "text": "will do you will fit a scaling law show"
+    },
+    {
+      "start": 2926.68,
+      "duration": 0.0,
+      "text": "will do you will fit a scaling law show another<00:48:46.920><c> scaling</c><00:48:47.280><c> law</c><00:48:47.520><c> with</c><00:48:47.680><c> the</c><00:48:47.839><c> standard</c>"
+    },
+    {
+      "start": 2928.27,
+      "duration": 0.0,
+      "text": "another scaling law with the standard"
+    },
+    {
+      "start": 2928.28,
+      "duration": 0.0,
+      "text": "another scaling law with the standard like<00:48:48.480><c> I</c><00:48:48.520><c> don't</c><00:48:48.640><c> know</c><00:48:48.839><c> G</c><00:48:49.559><c> and</c><00:48:49.680><c> you</c><00:48:49.760><c> will</c><00:48:49.880><c> say</c>"
+    },
+    {
+      "start": 2930.069,
+      "duration": 0.0,
+      "text": "like I don't know G and you will say"
+    },
+    {
+      "start": 2930.079,
+      "duration": 0.0,
+      "text": "like I don't know G and you will say that<00:48:50.200><c> it's</c><00:48:50.400><c> better</c><00:48:51.040><c> in</c><00:48:51.200><c> reality</c><00:48:51.559><c> once</c><00:48:51.720><c> you</c>"
+    },
+    {
+      "start": 2931.829,
+      "duration": 0.0,
+      "text": "that it's better in reality once you"
+    },
+    {
+      "start": 2931.839,
+      "duration": 0.0,
+      "text": "that it's better in reality once you start<00:48:52.079><c> thinking</c><00:48:52.319><c> about</c><00:48:52.480><c> it</c><00:48:52.599><c> in</c><00:48:52.720><c> scaling</c><00:48:53.079><c> loss</c>"
+    },
+    {
+      "start": 2933.39,
+      "duration": 0.0,
+      "text": "start thinking about it in scaling loss"
+    },
+    {
+      "start": 2933.4,
+      "duration": 0.0,
+      "text": "start thinking about it in scaling loss terms<00:48:53.960><c> you</c><00:48:54.160><c> really</c><00:48:54.440><c> realize</c><00:48:54.880><c> that</c><00:48:55.240><c> actually</c>"
+    },
+    {
+      "start": 2935.87,
+      "duration": 0.0,
+      "text": "terms you really realize that actually"
+    },
+    {
+      "start": 2935.88,
+      "duration": 0.0,
+      "text": "terms you really realize that actually all<00:48:56.079><c> the</c><00:48:56.200><c> architecture</c><00:48:56.760><c> differences</c><00:48:57.079><c> that</c><00:48:57.200><c> we</c>"
+    },
+    {
+      "start": 2937.27,
+      "duration": 0.0,
+      "text": "all the architecture differences that we"
+    },
+    {
+      "start": 2937.28,
+      "duration": 0.0,
+      "text": "all the architecture differences that we can<00:48:57.440><c> make</c><00:48:57.640><c> like</c><00:48:57.760><c> the</c><00:48:57.880><c> small</c><00:48:58.160><c> minor</c><00:48:58.520><c> ones</c><00:48:59.119><c> all</c>"
+    },
+    {
+      "start": 2939.309,
+      "duration": 0.0,
+      "text": "can make like the small minor ones all"
+    },
+    {
+      "start": 2939.319,
+      "duration": 0.0,
+      "text": "can make like the small minor ones all they<00:48:59.480><c> do</c><00:48:59.680><c> is</c><00:48:59.799><c> maybe</c><00:49:00.079><c> change</c><00:49:00.359><c> a</c><00:49:00.520><c> little</c><00:49:00.799><c> bit</c><00:49:01.119><c> the</c>"
+    },
+    {
+      "start": 2941.589,
+      "duration": 0.0,
+      "text": "they do is maybe change a little bit the"
+    },
+    {
+      "start": 2941.599,
+      "duration": 0.0,
+      "text": "they do is maybe change a little bit the The"
+    },
+    {
+      "start": 2942.43,
+      "duration": 0.0,
+      "text": "The"
+    },
+    {
+      "start": 2942.44,
+      "duration": 0.0,
+      "text": "The Intercept<00:49:03.440><c> but</c><00:49:03.640><c> really</c><00:49:03.839><c> that</c><00:49:04.000><c> doesn't</c><00:49:04.280><c> matter</c>"
+    },
+    {
+      "start": 2945.069,
+      "duration": 0.0,
+      "text": "Intercept but really that doesn't matter"
+    },
+    {
+      "start": 2945.079,
+      "duration": 0.0,
+      "text": "Intercept but really that doesn't matter uh<00:49:05.280><c> cuz</c><00:49:05.440><c> just</c><00:49:05.599><c> train</c><00:49:05.839><c> it</c><00:49:05.960><c> for</c><00:49:06.119><c> 10</c><00:49:06.319><c> hours</c><00:49:06.559><c> longer</c>"
+    },
+    {
+      "start": 2947.19,
+      "duration": 0.0,
+      "text": "uh cuz just train it for 10 hours longer"
+    },
+    {
+      "start": 2947.2,
+      "duration": 0.0,
+      "text": "uh cuz just train it for 10 hours longer or<00:49:07.400><c> like</c><00:49:07.640><c> wait</c><00:49:07.880><c> for</c><00:49:08.079><c> the</c><00:49:08.240><c> next</c><00:49:08.680><c> uh</c><00:49:08.839><c> for</c><00:49:09.000><c> the</c>"
+    },
+    {
+      "start": 2949.109,
+      "duration": 0.0,
+      "text": "or like wait for the next uh for the"
+    },
+    {
+      "start": 2949.119,
+      "duration": 0.0,
+      "text": "or like wait for the next uh for the next<00:49:09.359><c> Compu</c><00:49:09.760><c> gpus</c><00:49:10.359><c> and</c><00:49:10.520><c> these</c><00:49:10.680><c> things</c><00:49:10.920><c> are</c>"
+    },
+    {
+      "start": 2951.19,
+      "duration": 0.0,
+      "text": "next Compu gpus and these things are"
+    },
+    {
+      "start": 2951.2,
+      "duration": 0.0,
+      "text": "next Compu gpus and these things are really<00:49:11.480><c> secondary</c><00:49:12.079><c> which</c><00:49:12.200><c> is</c><00:49:12.319><c> exactly</c><00:49:12.640><c> why</c><00:49:12.760><c> I</c>"
+    },
+    {
+      "start": 2952.829,
+      "duration": 0.0,
+      "text": "really secondary which is exactly why I"
+    },
+    {
+      "start": 2952.839,
+      "duration": 0.0,
+      "text": "really secondary which is exactly why I was<00:49:12.960><c> telling</c><00:49:13.200><c> you</c><00:49:13.359><c> originally</c><00:49:14.000><c> people</c><00:49:14.280><c> spend</c>"
+    },
+    {
+      "start": 2954.549,
+      "duration": 0.0,
+      "text": "was telling you originally people spend"
+    },
+    {
+      "start": 2954.559,
+      "duration": 0.0,
+      "text": "was telling you originally people spend too<00:49:14.680><c> much</c><00:49:14.839><c> time</c><00:49:14.960><c> on</c><00:49:15.119><c> the</c><00:49:15.280><c> architecture</c><00:49:15.720><c> and</c>"
+    },
+    {
+      "start": 2955.829,
+      "duration": 0.0,
+      "text": "too much time on the architecture and"
+    },
+    {
+      "start": 2955.839,
+      "duration": 0.0,
+      "text": "too much time on the architecture and losses<00:49:16.799><c> um</c><00:49:17.240><c> in</c><00:49:17.400><c> reality</c><00:49:17.799><c> these</c><00:49:17.920><c> things</c><00:49:18.160><c> don't</c>"
+    },
+    {
+      "start": 2958.309,
+      "duration": 0.0,
+      "text": "losses um in reality these things don't"
+    },
+    {
+      "start": 2958.319,
+      "duration": 0.0,
+      "text": "losses um in reality these things don't matter<00:49:18.559><c> as</c><00:49:18.720><c> much</c><00:49:19.079><c> data</c><00:49:19.520><c> though</c><00:49:19.880><c> if</c><00:49:19.960><c> you</c><00:49:20.079><c> use</c>"
+    },
+    {
+      "start": 2960.309,
+      "duration": 0.0,
+      "text": "matter as much data though if you use"
+    },
+    {
+      "start": 2960.319,
+      "duration": 0.0,
+      "text": "matter as much data though if you use good<00:49:20.520><c> data</c><00:49:21.079><c> you</c><00:49:21.200><c> will</c><00:49:21.440><c> have</c><00:49:21.799><c> much</c><00:49:22.079><c> better</c>"
+    },
+    {
+      "start": 2962.27,
+      "duration": 0.0,
+      "text": "good data you will have much better"
+    },
+    {
+      "start": 2962.28,
+      "duration": 0.0,
+      "text": "good data you will have much better scaling<00:49:22.680><c> loss</c><00:49:23.119><c> than</c><00:49:23.240><c> if</c><00:49:23.440><c> use</c><00:49:23.760><c> bad</c><00:49:23.960><c> data</c><00:49:24.359><c> so</c>"
+    },
+    {
+      "start": 2964.549,
+      "duration": 0.0,
+      "text": "scaling loss than if use bad data so"
+    },
+    {
+      "start": 2964.559,
+      "duration": 0.0,
+      "text": "scaling loss than if use bad data so that<00:49:24.799><c> really</c><00:49:25.079><c> matters</c>"
+    },
+    {
+      "start": 2967.309,
+      "duration": 0.0,
+      "text": "that really matters"
+    },
+    {
+      "start": 2967.319,
+      "duration": 0.0,
+      "text": "that really matters uh<00:49:27.520><c> another</c><00:49:27.880><c> really</c><00:49:28.079><c> cool</c><00:49:28.280><c> thing</c><00:49:28.440><c> you</c><00:49:28.520><c> can</c><00:49:28.640><c> do</c>"
+    },
+    {
+      "start": 2968.71,
+      "duration": 0.0,
+      "text": "uh another really cool thing you can do"
+    },
+    {
+      "start": 2968.72,
+      "duration": 0.0,
+      "text": "uh another really cool thing you can do with<00:49:28.880><c> scaling</c><00:49:29.200><c> laws</c><00:49:29.720><c> is</c><00:49:29.880><c> that</c><00:49:30.000><c> you</c><00:49:30.079><c> can</c><00:49:30.280><c> ask</c>"
+    },
+    {
+      "start": 2970.549,
+      "duration": 0.0,
+      "text": "with scaling laws is that you can ask"
+    },
+    {
+      "start": 2970.559,
+      "duration": 0.0,
+      "text": "with scaling laws is that you can ask yourself<00:49:31.559><c> uh</c><00:49:32.160><c> how</c><00:49:32.359><c> to</c><00:49:32.760><c> optimally</c><00:49:33.400><c> allocate</c>"
+    },
+    {
+      "start": 2973.829,
+      "duration": 0.0,
+      "text": "yourself uh how to optimally allocate"
+    },
+    {
+      "start": 2973.839,
+      "duration": 0.0,
+      "text": "yourself uh how to optimally allocate training<00:49:34.240><c> resources</c><00:49:35.079><c> should</c><00:49:35.319><c> I</c><00:49:35.559><c> train</c><00:49:35.960><c> larger</c>"
+    },
+    {
+      "start": 2976.349,
+      "duration": 0.0,
+      "text": "training resources should I train larger"
+    },
+    {
+      "start": 2976.359,
+      "duration": 0.0,
+      "text": "training resources should I train larger models<00:49:37.000><c> because</c><00:49:37.119><c> we</c><00:49:37.240><c> saw</c><00:49:37.520><c> that</c><00:49:37.960><c> it's</c><00:49:38.160><c> better</c>"
+    },
+    {
+      "start": 2978.39,
+      "duration": 0.0,
+      "text": "models because we saw that it's better"
+    },
+    {
+      "start": 2978.4,
+      "duration": 0.0,
+      "text": "models because we saw that it's better when<00:49:38.520><c> you</c><00:49:38.599><c> train</c><00:49:38.880><c> larger</c><00:49:39.119><c> models</c><00:49:39.640><c> but</c><00:49:39.799><c> we</c><00:49:39.920><c> saw</c>"
+    },
+    {
+      "start": 2980.109,
+      "duration": 0.0,
+      "text": "when you train larger models but we saw"
+    },
+    {
+      "start": 2980.119,
+      "duration": 0.0,
+      "text": "when you train larger models but we saw that<00:49:40.240><c> it's</c><00:49:40.359><c> also</c><00:49:40.559><c> better</c><00:49:40.799><c> when</c><00:49:40.920><c> you</c><00:49:41.040><c> use</c><00:49:41.480><c> more</c>"
+    },
+    {
+      "start": 2981.71,
+      "duration": 0.0,
+      "text": "that it's also better when you use more"
+    },
+    {
+      "start": 2981.72,
+      "duration": 0.0,
+      "text": "that it's also better when you use more data<00:49:42.319><c> so</c><00:49:42.680><c> which</c><00:49:42.920><c> one</c><00:49:43.079><c> should</c><00:49:43.240><c> I</c><00:49:43.400><c> do</c><00:49:43.720><c> should</c><00:49:43.880><c> I</c>"
+    },
+    {
+      "start": 2984.03,
+      "duration": 0.0,
+      "text": "data so which one should I do should I"
+    },
+    {
+      "start": 2984.04,
+      "duration": 0.0,
+      "text": "data so which one should I do should I just<00:49:44.160><c> train</c><00:49:44.400><c> on</c><00:49:44.520><c> more</c><00:49:44.720><c> data</c><00:49:45.040><c> a</c><00:49:45.160><c> smaller</c><00:49:45.480><c> model</c>"
+    },
+    {
+      "start": 2985.87,
+      "duration": 0.0,
+      "text": "just train on more data a smaller model"
+    },
+    {
+      "start": 2985.88,
+      "duration": 0.0,
+      "text": "just train on more data a smaller model or<00:49:46.000><c> should</c><00:49:46.160><c> I</c><00:49:46.319><c> train</c><00:49:46.559><c> a</c><00:49:46.680><c> larger</c><00:49:46.960><c> model</c><00:49:47.240><c> on</c><00:49:47.440><c> less</c>"
+    },
+    {
+      "start": 2987.87,
+      "duration": 0.0,
+      "text": "or should I train a larger model on less"
+    },
+    {
+      "start": 2987.88,
+      "duration": 0.0,
+      "text": "or should I train a larger model on less data<00:49:48.880><c> um</c><00:49:49.760><c> so</c><00:49:50.760><c> chinchilla</c><00:49:51.319><c> is</c><00:49:51.400><c> a</c><00:49:51.520><c> very</c><00:49:51.720><c> famous</c>"
+    },
+    {
+      "start": 2991.99,
+      "duration": 0.0,
+      "text": "data um so chinchilla is a very famous"
+    },
+    {
+      "start": 2992.0,
+      "duration": 0.0,
+      "text": "data um so chinchilla is a very famous paper<00:49:52.319><c> that</c><00:49:52.520><c> first</c><00:49:52.720><c> showed</c><00:49:53.119><c> this</c><00:49:53.799><c> uh</c><00:49:53.960><c> the</c><00:49:54.079><c> way</c>"
+    },
+    {
+      "start": 2994.23,
+      "duration": 0.0,
+      "text": "paper that first showed this uh the way"
+    },
+    {
+      "start": 2994.24,
+      "duration": 0.0,
+      "text": "paper that first showed this uh the way they<00:49:54.440><c> did</c><00:49:54.599><c> it</c><00:49:55.079><c> I</c><00:49:55.160><c> want</c><00:49:55.280><c> to</c><00:49:55.440><c> give</c><00:49:55.520><c> you</c><00:49:55.640><c> a</c><00:49:55.760><c> little</c>"
+    },
+    {
+      "start": 2995.91,
+      "duration": 0.0,
+      "text": "they did it I want to give you a little"
+    },
+    {
+      "start": 2995.92,
+      "duration": 0.0,
+      "text": "they did it I want to give you a little bit<00:49:56.280><c> of</c><00:49:56.440><c> a</c><00:49:56.720><c> sense</c><00:49:56.880><c> of</c><00:49:57.000><c> what</c><00:49:57.079><c> these</c><00:49:57.240><c> plots</c><00:49:57.559><c> are</c>"
+    },
+    {
+      "start": 2998.349,
+      "duration": 0.0,
+      "text": "bit of a sense of what these plots are"
+    },
+    {
+      "start": 2998.359,
+      "duration": 0.0,
+      "text": "bit of a sense of what these plots are uh<00:49:58.480><c> here</c><00:49:58.599><c> you</c><00:49:58.720><c> see</c><00:49:58.839><c> training</c><00:49:59.119><c> loss</c><00:49:59.520><c> again</c><00:49:59.880><c> on</c>"
+    },
+    {
+      "start": 2999.99,
+      "duration": 0.0,
+      "text": "uh here you see training loss again on"
+    },
+    {
+      "start": 3000.0,
+      "duration": 0.0,
+      "text": "uh here you see training loss again on the<00:50:00.160><c> x-axis</c><00:50:00.720><c> you</c><00:50:00.799><c> see</c><00:50:01.079><c> parameter</c><00:50:01.799><c> parameter</c>"
+    },
+    {
+      "start": 3002.19,
+      "duration": 0.0,
+      "text": "the x-axis you see parameter parameter"
+    },
+    {
+      "start": 3002.2,
+      "duration": 0.0,
+      "text": "the x-axis you see parameter parameter differences<00:50:02.799><c> uh</c><00:50:02.920><c> sorry</c><00:50:03.160><c> parameter</c><00:50:03.520><c> size</c><00:50:03.960><c> uh</c>"
+    },
+    {
+      "start": 3004.069,
+      "duration": 0.0,
+      "text": "differences uh sorry parameter size uh"
+    },
+    {
+      "start": 3004.079,
+      "duration": 0.0,
+      "text": "differences uh sorry parameter size uh number<00:50:04.280><c> of</c><00:50:04.400><c> parameters</c><00:50:04.799><c> so</c><00:50:04.960><c> the</c><00:50:05.079><c> size</c><00:50:05.240><c> of</c><00:50:05.359><c> the</c>"
+    },
+    {
+      "start": 3005.47,
+      "duration": 0.0,
+      "text": "number of parameters so the size of the"
+    },
+    {
+      "start": 3005.48,
+      "duration": 0.0,
+      "text": "number of parameters so the size of the model<00:50:06.280><c> and</c><00:50:06.520><c> here</c><00:50:06.799><c> all</c><00:50:07.000><c> these</c><00:50:07.240><c> curves</c><00:50:07.559><c> are</c><00:50:07.720><c> what</c>"
+    },
+    {
+      "start": 3007.829,
+      "duration": 0.0,
+      "text": "model and here all these curves are what"
+    },
+    {
+      "start": 3007.839,
+      "duration": 0.0,
+      "text": "model and here all these curves are what we<00:50:08.000><c> call</c><00:50:08.240><c> isof</c><00:50:08.680><c> flops</c><00:50:09.319><c> which</c><00:50:09.480><c> is</c><00:50:10.200><c> that</c><00:50:10.760><c> all</c><00:50:11.319><c> the</c>"
+    },
+    {
+      "start": 3011.47,
+      "duration": 0.0,
+      "text": "we call isof flops which is that all the"
+    },
+    {
+      "start": 3011.48,
+      "duration": 0.0,
+      "text": "we call isof flops which is that all the models<00:50:12.079><c> on</c><00:50:12.400><c> this</c><00:50:12.720><c> curve</c><00:50:13.720><c> H</c><00:50:14.000><c> have</c><00:50:14.160><c> been</c><00:50:14.319><c> trained</c>"
+    },
+    {
+      "start": 3014.67,
+      "duration": 0.0,
+      "text": "models on this curve H have been trained"
+    },
+    {
+      "start": 3014.68,
+      "duration": 0.0,
+      "text": "models on this curve H have been trained with<00:50:14.839><c> the</c><00:50:14.920><c> same</c><00:50:15.119><c> amount</c><00:50:15.359><c> of</c>"
+    },
+    {
+      "start": 3015.99,
+      "duration": 0.0,
+      "text": "with the same amount of"
+    },
+    {
+      "start": 3016.0,
+      "duration": 0.0,
+      "text": "with the same amount of compute<00:50:17.000><c> um</c><00:50:17.240><c> the</c><00:50:17.359><c> way</c><00:50:17.520><c> that</c><00:50:17.640><c> you</c><00:50:17.799><c> do</c><00:50:18.000><c> that</c><00:50:18.280><c> is</c>"
+    },
+    {
+      "start": 3018.349,
+      "duration": 0.0,
+      "text": "compute um the way that you do that is"
+    },
+    {
+      "start": 3018.359,
+      "duration": 0.0,
+      "text": "compute um the way that you do that is that<00:50:18.520><c> you</c><00:50:18.640><c> train</c><00:50:19.200><c> you</c><00:50:19.559><c> change</c><00:50:20.119><c> sorry</c><00:50:20.400><c> you</c><00:50:20.520><c> vary</c>"
+    },
+    {
+      "start": 3020.789,
+      "duration": 0.0,
+      "text": "that you train you change sorry you vary"
+    },
+    {
+      "start": 3020.799,
+      "duration": 0.0,
+      "text": "that you train you change sorry you vary the<00:50:20.880><c> number</c><00:50:21.079><c> of</c><00:50:21.200><c> tokens</c><00:50:21.520><c> that</c><00:50:21.640><c> we</c><00:50:21.839><c> trained</c><00:50:22.119><c> on</c>"
+    },
+    {
+      "start": 3022.589,
+      "duration": 0.0,
+      "text": "the number of tokens that we trained on"
+    },
+    {
+      "start": 3022.599,
+      "duration": 0.0,
+      "text": "the number of tokens that we trained on and<00:50:22.720><c> the</c><00:50:22.880><c> size</c><00:50:23.079><c> of</c><00:50:23.200><c> the</c><00:50:23.319><c> models</c><00:50:23.920><c> but</c><00:50:24.040><c> you</c><00:50:24.200><c> vary</c>"
+    },
+    {
+      "start": 3024.43,
+      "duration": 0.0,
+      "text": "and the size of the models but you vary"
+    },
+    {
+      "start": 3024.44,
+      "duration": 0.0,
+      "text": "and the size of the models but you vary in<00:50:24.559><c> such</c><00:50:24.720><c> a</c><00:50:24.799><c> way</c><00:50:24.960><c> that</c><00:50:25.079><c> the</c><00:50:25.240><c> total</c><00:50:25.559><c> compute</c><00:50:26.119><c> is</c>"
+    },
+    {
+      "start": 3026.309,
+      "duration": 0.0,
+      "text": "in such a way that the total compute is"
+    },
+    {
+      "start": 3026.319,
+      "duration": 0.0,
+      "text": "in such a way that the total compute is constant"
+    },
+    {
+      "start": 3027.27,
+      "duration": 0.0,
+      "text": "constant"
+    },
+    {
+      "start": 3027.28,
+      "duration": 0.0,
+      "text": "constant okay<00:50:27.640><c> so</c><00:50:27.920><c> all</c><00:50:28.079><c> these</c><00:50:28.319><c> curves</c><00:50:28.599><c> that</c><00:50:28.680><c> you</c><00:50:28.760><c> see</c>"
+    },
+    {
+      "start": 3028.91,
+      "duration": 0.0,
+      "text": "okay so all these curves that you see"
+    },
+    {
+      "start": 3028.92,
+      "duration": 0.0,
+      "text": "okay so all these curves that you see with<00:50:29.079><c> different</c><00:50:29.280><c> colors</c><00:50:30.040><c> have</c><00:50:30.319><c> different</c>"
+    },
+    {
+      "start": 3030.63,
+      "duration": 0.0,
+      "text": "with different colors have different"
+    },
+    {
+      "start": 3030.64,
+      "duration": 0.0,
+      "text": "with different colors have different amount<00:50:30.880><c> of</c><00:50:31.000><c> computers</c><00:50:31.400><c> that</c><00:50:31.520><c> were</c><00:50:31.680><c> trained</c><00:50:31.960><c> on</c>"
+    },
+    {
+      "start": 3032.71,
+      "duration": 0.0,
+      "text": "amount of computers that were trained on"
+    },
+    {
+      "start": 3032.72,
+      "duration": 0.0,
+      "text": "amount of computers that were trained on then<00:50:32.880><c> you</c><00:50:33.040><c> take</c><00:50:33.200><c> the</c><00:50:33.359><c> best</c><00:50:33.599><c> one</c><00:50:33.839><c> for</c><00:50:34.079><c> each</c><00:50:34.240><c> of</c>"
+    },
+    {
+      "start": 3034.39,
+      "duration": 0.0,
+      "text": "then you take the best one for each of"
+    },
+    {
+      "start": 3034.4,
+      "duration": 0.0,
+      "text": "then you take the best one for each of those<00:50:34.599><c> curves</c><00:50:35.559><c> once</c><00:50:35.720><c> you</c><00:50:35.880><c> have</c><00:50:36.000><c> the</c><00:50:36.200><c> best</c><00:50:36.359><c> one</c>"
+    },
+    {
+      "start": 3036.51,
+      "duration": 0.0,
+      "text": "those curves once you have the best one"
+    },
+    {
+      "start": 3036.52,
+      "duration": 0.0,
+      "text": "those curves once you have the best one for<00:50:36.720><c> each</c><00:50:36.880><c> of</c><00:50:37.040><c> those</c><00:50:37.400><c> curves</c><00:50:38.400><c> um</c><00:50:38.880><c> you</c><00:50:39.319><c> can</c><00:50:40.319><c> ask</c>"
+    },
+    {
+      "start": 3040.71,
+      "duration": 0.0,
+      "text": "for each of those curves um you can ask"
+    },
+    {
+      "start": 3040.72,
+      "duration": 0.0,
+      "text": "for each of those curves um you can ask you<00:50:40.839><c> can</c><00:50:41.400><c> plot</c><00:50:42.400><c> um</c><00:50:42.880><c> how</c><00:50:43.000><c> much</c><00:50:43.200><c> flops</c><00:50:43.640><c> it</c><00:50:43.799><c> was</c>"
+    },
+    {
+      "start": 3044.03,
+      "duration": 0.0,
+      "text": "you can plot um how much flops it was"
+    },
+    {
+      "start": 3044.04,
+      "duration": 0.0,
+      "text": "you can plot um how much flops it was and<00:50:44.400><c> which</c><00:50:44.599><c> curve</c><00:50:44.880><c> were</c><00:50:45.040><c> you</c><00:50:45.200><c> on</c><00:50:45.799><c> and</c><00:50:46.000><c> how</c><00:50:46.119><c> much</c>"
+    },
+    {
+      "start": 3046.47,
+      "duration": 0.0,
+      "text": "and which curve were you on and how much"
+    },
+    {
+      "start": 3046.48,
+      "duration": 0.0,
+      "text": "and which curve were you on and how much parameters<00:50:47.480><c> did</c><00:50:47.640><c> you</c><00:50:47.880><c> actually</c><00:50:48.200><c> use</c><00:50:48.920><c> for</c>"
+    },
+    {
+      "start": 3049.15,
+      "duration": 0.0,
+      "text": "parameters did you actually use for"
+    },
+    {
+      "start": 3049.16,
+      "duration": 0.0,
+      "text": "parameters did you actually use for training<00:50:49.640><c> that</c><00:50:49.880><c> specific</c><00:50:50.280><c> point</c><00:50:50.839><c> you</c><00:50:51.040><c> put</c>"
+    },
+    {
+      "start": 3051.309,
+      "duration": 0.0,
+      "text": "training that specific point you put"
+    },
+    {
+      "start": 3051.319,
+      "duration": 0.0,
+      "text": "training that specific point you put that<00:50:51.839><c> on</c><00:50:52.040><c> the</c><00:50:52.440><c> on</c><00:50:52.559><c> the</c><00:50:52.720><c> log</c><00:50:53.040><c> log</c><00:50:53.760><c> uh</c><00:50:53.920><c> scale</c>"
+    },
+    {
+      "start": 3054.51,
+      "duration": 0.0,
+      "text": "that on the on the log log uh scale"
+    },
+    {
+      "start": 3054.52,
+      "duration": 0.0,
+      "text": "that on the on the log log uh scale again<00:50:54.760><c> and</c><00:50:54.880><c> now</c><00:50:55.000><c> you</c><00:50:55.119><c> fit</c><00:50:55.319><c> a</c><00:50:55.480><c> scaling</c><00:50:55.839><c> law</c>"
+    },
+    {
+      "start": 3056.39,
+      "duration": 0.0,
+      "text": "again and now you fit a scaling law"
+    },
+    {
+      "start": 3056.4,
+      "duration": 0.0,
+      "text": "again and now you fit a scaling law again<00:50:56.960><c> so</c><00:50:57.200><c> now</c><00:50:57.400><c> I</c><00:50:57.599><c> have</c><00:50:58.319><c> something</c><00:50:58.760><c> which</c>"
+    },
+    {
+      "start": 3058.95,
+      "duration": 0.0,
+      "text": "again so now I have something which"
+    },
+    {
+      "start": 3058.96,
+      "duration": 0.0,
+      "text": "again so now I have something which tells<00:50:59.240><c> me</c><00:50:59.920><c> if</c><00:51:00.119><c> I</c><00:51:00.200><c> want</c><00:51:00.359><c> to</c><00:51:00.520><c> train</c><00:51:00.839><c> a</c><00:51:00.960><c> model</c><00:51:01.240><c> of</c>"
+    },
+    {
+      "start": 3061.39,
+      "duration": 0.0,
+      "text": "tells me if I want to train a model of"
+    },
+    {
+      "start": 3061.4,
+      "duration": 0.0,
+      "text": "tells me if I want to train a model of 10^<00:51:02.040><c> 23</c><00:51:02.480><c> flops</c><00:51:03.440><c> here's</c><00:51:03.760><c> exactly</c><00:51:04.119><c> the</c><00:51:04.240><c> number</c>"
+    },
+    {
+      "start": 3064.43,
+      "duration": 0.0,
+      "text": "10^ 23 flops here's exactly the number"
+    },
+    {
+      "start": 3064.44,
+      "duration": 0.0,
+      "text": "10^ 23 flops here's exactly the number of<00:51:04.599><c> parameters</c><00:51:04.960><c> that</c><00:51:05.079><c> I</c><00:51:05.160><c> should</c><00:51:05.319><c> be</c><00:51:05.440><c> using</c><00:51:06.240><c> 100</c>"
+    },
+    {
+      "start": 3066.99,
+      "duration": 0.0,
+      "text": "of parameters that I should be using 100"
+    },
+    {
+      "start": 3067.0,
+      "duration": 0.0,
+      "text": "of parameters that I should be using 100 100b<00:51:07.960><c> and</c><00:51:08.079><c> you</c><00:51:08.160><c> can</c><00:51:08.280><c> do</c><00:51:08.440><c> the</c><00:51:08.559><c> same</c><00:51:08.760><c> thing</c><00:51:08.920><c> with</c>"
+    },
+    {
+      "start": 3069.03,
+      "duration": 0.0,
+      "text": "100b and you can do the same thing with"
+    },
+    {
+      "start": 3069.04,
+      "duration": 0.0,
+      "text": "100b and you can do the same thing with flops<00:51:09.599><c> and</c>"
+    },
+    {
+      "start": 3070.51,
+      "duration": 0.0,
+      "text": "flops and"
+    },
+    {
+      "start": 3070.52,
+      "duration": 0.0,
+      "text": "flops and tokens<00:51:11.520><c> so</c><00:51:11.680><c> now</c><00:51:11.799><c> you</c><00:51:11.920><c> can</c><00:51:12.559><c> predict</c><00:51:13.559><c> if</c><00:51:13.799><c> if</c><00:51:13.960><c> I</c>"
+    },
+    {
+      "start": 3074.069,
+      "duration": 0.0,
+      "text": "tokens so now you can predict if if I"
+    },
+    {
+      "start": 3074.079,
+      "duration": 0.0,
+      "text": "tokens so now you can predict if if I tell<00:51:14.280><c> you</c><00:51:14.480><c> exactly</c><00:51:14.799><c> I</c><00:51:14.920><c> have</c><00:51:15.040><c> one</c><00:51:15.200><c> month</c><00:51:15.440><c> of</c>"
+    },
+    {
+      "start": 3076.03,
+      "duration": 0.0,
+      "text": "tell you exactly I have one month of"
+    },
+    {
+      "start": 3076.04,
+      "duration": 0.0,
+      "text": "tell you exactly I have one month of compute<00:51:17.040><c> what</c><00:51:17.240><c> size</c><00:51:17.440><c> of</c><00:51:17.599><c> model</c><00:51:17.839><c> should</c><00:51:18.000><c> I</c><00:51:18.079><c> be</c>"
+    },
+    {
+      "start": 3078.19,
+      "duration": 0.0,
+      "text": "compute what size of model should I be"
+    },
+    {
+      "start": 3078.2,
+      "duration": 0.0,
+      "text": "compute what size of model should I be training<00:51:18.839><c> F</c><00:51:19.119><c> your</c><00:51:19.240><c> scaling</c><00:51:19.599><c> law</c><00:51:19.880><c> and</c><00:51:20.000><c> I</c><00:51:20.119><c> tell</c>"
+    },
+    {
+      "start": 3080.589,
+      "duration": 0.0,
+      "text": "training F your scaling law and I tell"
+    },
+    {
+      "start": 3080.599,
+      "duration": 0.0,
+      "text": "training F your scaling law and I tell you<00:51:21.599><c> um</c><00:51:22.119><c> of</c><00:51:22.240><c> course</c><00:51:22.440><c> that</c><00:51:22.599><c> all</c><00:51:22.760><c> looks</c>"
+    },
+    {
+      "start": 3083.03,
+      "duration": 0.0,
+      "text": "you um of course that all looks"
+    },
+    {
+      "start": 3083.04,
+      "duration": 0.0,
+      "text": "you um of course that all looks beautiful<00:51:23.760><c> in</c><00:51:23.960><c> reality</c><00:51:24.520><c> like</c><00:51:24.760><c> there's</c><00:51:25.119><c> like</c>"
+    },
+    {
+      "start": 3085.23,
+      "duration": 0.0,
+      "text": "beautiful in reality like there's like"
+    },
+    {
+      "start": 3085.24,
+      "duration": 0.0,
+      "text": "beautiful in reality like there's like there's<00:51:25.400><c> a</c><00:51:25.559><c> lot</c><00:51:25.680><c> of</c><00:51:25.880><c> like</c><00:51:26.000><c> small</c><00:51:26.280><c> things</c><00:51:26.680><c> of</c>"
+    },
+    {
+      "start": 3086.789,
+      "duration": 0.0,
+      "text": "there's a lot of like small things of"
+    },
+    {
+      "start": 3086.799,
+      "duration": 0.0,
+      "text": "there's a lot of like small things of like<00:51:26.920><c> should</c><00:51:27.040><c> you</c><00:51:27.160><c> be</c><00:51:27.319><c> counting</c><00:51:27.720><c> like</c>"
+    },
+    {
+      "start": 3087.829,
+      "duration": 0.0,
+      "text": "like should you be counting like"
+    },
+    {
+      "start": 3087.839,
+      "duration": 0.0,
+      "text": "like should you be counting like embedding<00:51:28.319><c> parameters</c><00:51:29.160><c> like</c><00:51:29.359><c> there's</c>"
+    },
+    {
+      "start": 3089.549,
+      "duration": 0.0,
+      "text": "embedding parameters like there's"
+    },
+    {
+      "start": 3089.559,
+      "duration": 0.0,
+      "text": "embedding parameters like there's there's<00:51:29.680><c> a</c><00:51:29.799><c> lot</c><00:51:29.880><c> of</c><00:51:30.040><c> complexities</c><00:51:31.040><c> but</c><00:51:31.200><c> if</c><00:51:31.280><c> you</c>"
+    },
+    {
+      "start": 3091.43,
+      "duration": 0.0,
+      "text": "there's a lot of complexities but if you"
+    },
+    {
+      "start": 3091.44,
+      "duration": 0.0,
+      "text": "there's a lot of complexities but if you do<00:51:31.640><c> things</c><00:51:31.920><c> well</c><00:51:32.319><c> these</c><00:51:32.480><c> things</c><00:51:32.720><c> actually</c><00:51:33.000><c> do</c>"
+    },
+    {
+      "start": 3093.99,
+      "duration": 0.0,
+      "text": "do things well these things actually do"
+    },
+    {
+      "start": 3094.0,
+      "duration": 0.0,
+      "text": "do things well these things actually do hold<00:51:35.000><c> um</c><00:51:35.640><c> so</c><00:51:35.960><c> the</c><00:51:36.240><c> optimal</c><00:51:36.760><c> number</c><00:51:37.000><c> of</c>"
+    },
+    {
+      "start": 3097.19,
+      "duration": 0.0,
+      "text": "hold um so the optimal number of"
+    },
+    {
+      "start": 3097.2,
+      "duration": 0.0,
+      "text": "hold um so the optimal number of parameters<00:51:37.720><c> that</c><00:51:38.000><c> that</c><00:51:38.119><c> chinchilla</c><00:51:38.640><c> Pap</c><00:51:39.000><c> have</c>"
+    },
+    {
+      "start": 3099.15,
+      "duration": 0.0,
+      "text": "parameters that that chinchilla Pap have"
+    },
+    {
+      "start": 3099.16,
+      "duration": 0.0,
+      "text": "parameters that that chinchilla Pap have found<00:51:39.640><c> is</c><00:51:39.760><c> to</c><00:51:40.000><c> use</c><00:51:40.839><c> 20</c><00:51:41.240><c> tokens</c><00:51:41.839><c> for</c><00:51:42.119><c> every</c>"
+    },
+    {
+      "start": 3102.349,
+      "duration": 0.0,
+      "text": "found is to use 20 tokens for every"
+    },
+    {
+      "start": 3102.359,
+      "duration": 0.0,
+      "text": "found is to use 20 tokens for every parameter<00:51:42.799><c> that</c><00:51:42.880><c> you</c><00:51:43.000><c> train</c><00:51:44.000><c> uh</c><00:51:44.079><c> so</c><00:51:44.240><c> if</c><00:51:44.319><c> you</c>"
+    },
+    {
+      "start": 3104.43,
+      "duration": 0.0,
+      "text": "parameter that you train uh so if you"
+    },
+    {
+      "start": 3104.44,
+      "duration": 0.0,
+      "text": "parameter that you train uh so if you add<00:51:44.640><c> one</c><00:51:44.760><c> more</c><00:51:44.920><c> parameter</c><00:51:45.440><c> you</c><00:51:45.520><c> should</c><00:51:45.799><c> add</c>"
+    },
+    {
+      "start": 3105.91,
+      "duration": 0.0,
+      "text": "add one more parameter you should add"
+    },
+    {
+      "start": 3105.92,
+      "duration": 0.0,
+      "text": "add one more parameter you should add you<00:51:46.000><c> should</c><00:51:46.200><c> train</c><00:51:46.440><c> your</c><00:51:46.720><c> thing</c><00:51:46.880><c> on</c><00:51:47.359><c> your</c>"
+    },
+    {
+      "start": 3107.47,
+      "duration": 0.0,
+      "text": "you should train your thing on your"
+    },
+    {
+      "start": 3107.48,
+      "duration": 0.0,
+      "text": "you should train your thing on your model<00:51:47.720><c> on</c><00:51:47.799><c> 20</c><00:51:48.040><c> more</c><00:51:48.640><c> tokens</c><00:51:49.640><c> so</c><00:51:49.880><c> one</c><00:51:50.280><c> caveat</c>"
+    },
+    {
+      "start": 3110.789,
+      "duration": 0.0,
+      "text": "model on 20 more tokens so one caveat"
+    },
+    {
+      "start": 3110.799,
+      "duration": 0.0,
+      "text": "model on 20 more tokens so one caveat here<00:51:51.000><c> is</c><00:51:51.119><c> that</c><00:51:51.280><c> this</c><00:51:51.400><c> is</c><00:51:51.599><c> optimal</c><00:51:52.000><c> training</c>"
+    },
+    {
+      "start": 3112.39,
+      "duration": 0.0,
+      "text": "here is that this is optimal training"
+    },
+    {
+      "start": 3112.4,
+      "duration": 0.0,
+      "text": "here is that this is optimal training resources<00:51:53.200><c> so</c><00:51:53.359><c> that</c><00:51:53.480><c> is</c><00:51:53.680><c> telling</c><00:51:53.960><c> me</c><00:51:54.280><c> if</c><00:51:54.400><c> you</c>"
+    },
+    {
+      "start": 3114.67,
+      "duration": 0.0,
+      "text": "resources so that is telling me if you"
+    },
+    {
+      "start": 3114.68,
+      "duration": 0.0,
+      "text": "resources so that is telling me if you have<00:51:55.240><c> 10^</c><00:51:55.880><c> 23</c><00:51:56.240><c> FL</c>"
+    },
+    {
+      "start": 3117.15,
+      "duration": 0.0,
+      "text": "have 10^ 23 FL"
+    },
+    {
+      "start": 3117.16,
+      "duration": 0.0,
+      "text": "have 10^ 23 FL or<00:51:57.359><c> if</c><00:51:57.440><c> you</c><00:51:57.559><c> have</c><00:51:57.720><c> like</c><00:51:57.960><c> 100</c><00:51:58.400><c> I</c><00:51:58.480><c> don't</c><00:51:58.599><c> know</c><00:51:58.799><c> how</c>"
+    },
+    {
+      "start": 3118.91,
+      "duration": 0.0,
+      "text": "or if you have like 100 I don't know how"
+    },
+    {
+      "start": 3118.92,
+      "duration": 0.0,
+      "text": "or if you have like 100 I don't know how much<00:51:59.119><c> that</c><00:51:59.240><c> is100</c><00:51:59.880><c> million</c><00:52:00.880><c> or</c><00:52:01.119><c> 10</c><00:52:01.400><c> no</c><00:52:01.720><c> that's</c>"
+    },
+    {
+      "start": 3121.99,
+      "duration": 0.0,
+      "text": "much that is100 million or 10 no that's"
+    },
+    {
+      "start": 3122.0,
+      "duration": 0.0,
+      "text": "much that is100 million or 10 no that's much<00:52:02.240><c> less</c><00:52:02.480><c> actually</c><00:52:02.799><c> let's</c><00:52:02.960><c> say</c><00:52:03.079><c> I</c><00:52:03.200><c> have</c><00:52:03.280><c> $5</c>"
+    },
+    {
+      "start": 3123.51,
+      "duration": 0.0,
+      "text": "much less actually let's say I have $5"
+    },
+    {
+      "start": 3123.52,
+      "duration": 0.0,
+      "text": "much less actually let's say I have $5 million<00:52:04.119><c> to</c><00:52:04.319><c> to</c><00:52:04.839><c> train</c><00:52:05.240><c> my</c><00:52:05.480><c> best</c><00:52:05.839><c> model</c><00:52:06.280><c> that</c>"
+    },
+    {
+      "start": 3126.39,
+      "duration": 0.0,
+      "text": "million to to train my best model that"
+    },
+    {
+      "start": 3126.4,
+      "duration": 0.0,
+      "text": "million to to train my best model that gets<00:52:06.599><c> the</c><00:52:06.720><c> lowest</c><00:52:07.040><c> loss</c><00:52:07.680><c> how</c><00:52:07.960><c> how</c><00:52:08.200><c> what</c><00:52:08.359><c> would</c>"
+    },
+    {
+      "start": 3128.51,
+      "duration": 0.0,
+      "text": "gets the lowest loss how how what would"
+    },
+    {
+      "start": 3128.52,
+      "duration": 0.0,
+      "text": "gets the lowest loss how how what would I<00:52:08.799><c> train</c><00:52:09.119><c> on</c><00:52:09.920><c> in</c><00:52:10.119><c> reality</c><00:52:10.599><c> these</c><00:52:10.799><c> companies</c>"
+    },
+    {
+      "start": 3131.109,
+      "duration": 0.0,
+      "text": "I train on in reality these companies"
+    },
+    {
+      "start": 3131.119,
+      "duration": 0.0,
+      "text": "I train on in reality these companies need<00:52:11.280><c> to</c><00:52:11.400><c> think</c><00:52:11.559><c> about</c><00:52:11.799><c> inference</c><00:52:12.240><c> also</c><00:52:12.920><c> if</c>"
+    },
+    {
+      "start": 3133.03,
+      "duration": 0.0,
+      "text": "need to think about inference also if"
+    },
+    {
+      "start": 3133.04,
+      "duration": 0.0,
+      "text": "need to think about inference also if you<00:52:13.160><c> have</c><00:52:13.240><c> a</c><00:52:13.359><c> smaller</c><00:52:13.760><c> model</c><00:52:14.760><c> they</c><00:52:14.920><c> will</c><00:52:15.559><c> spend</c>"
+    },
+    {
+      "start": 3135.99,
+      "duration": 0.0,
+      "text": "you have a smaller model they will spend"
+    },
+    {
+      "start": 3136.0,
+      "duration": 0.0,
+      "text": "you have a smaller model they will spend less<00:52:16.280><c> over</c><00:52:16.599><c> time</c><00:52:17.520><c> um</c><00:52:17.920><c> so</c><00:52:18.160><c> actually</c><00:52:18.640><c> if</c><00:52:18.720><c> you</c>"
+    },
+    {
+      "start": 3138.87,
+      "duration": 0.0,
+      "text": "less over time um so actually if you"
+    },
+    {
+      "start": 3138.88,
+      "duration": 0.0,
+      "text": "less over time um so actually if you consider<00:52:19.240><c> the</c><00:52:19.359><c> inference</c><00:52:19.799><c> cost</c><00:52:20.160><c> you</c><00:52:20.280><c> have</c>"
+    },
+    {
+      "start": 3140.39,
+      "duration": 0.0,
+      "text": "consider the inference cost you have"
+    },
+    {
+      "start": 3140.4,
+      "duration": 0.0,
+      "text": "consider the inference cost you have other<00:52:20.640><c> papers</c><00:52:20.920><c> that</c><00:52:21.040><c> Tred</c><00:52:21.240><c> to</c><00:52:21.359><c> show</c><00:52:21.640><c> that</c><00:52:22.319><c> um</c>"
+    },
+    {
+      "start": 3142.67,
+      "duration": 0.0,
+      "text": "other papers that Tred to show that um"
+    },
+    {
+      "start": 3142.68,
+      "duration": 0.0,
+      "text": "other papers that Tred to show that um it's<00:52:22.920><c> around</c>"
+    },
+    {
+      "start": 3143.99,
+      "duration": 0.0,
+      "text": "it's around"
+    },
+    {
+      "start": 3144.0,
+      "duration": 0.0,
+      "text": "it's around 150<00:52:25.000><c> uh</c><00:52:25.240><c> parameters</c><00:52:26.079><c> per</c><00:52:26.280><c> sorry</c><00:52:26.880><c> tokens</c><00:52:27.400><c> per</c>"
+    },
+    {
+      "start": 3147.549,
+      "duration": 0.0,
+      "text": "150 uh parameters per sorry tokens per"
+    },
+    {
+      "start": 3147.559,
+      "duration": 0.0,
+      "text": "150 uh parameters per sorry tokens per parameters<00:52:28.160><c> because</c><00:52:28.319><c> you</c><00:52:28.480><c> prefer</c><00:52:29.079><c> having</c><00:52:29.319><c> a</c>"
+    },
+    {
+      "start": 3149.47,
+      "duration": 0.0,
+      "text": "parameters because you prefer having a"
+    },
+    {
+      "start": 3149.48,
+      "duration": 0.0,
+      "text": "parameters because you prefer having a smaller<00:52:29.920><c> model</c><00:52:30.760><c> cuz</c><00:52:31.000><c> over</c><00:52:31.280><c> time</c><00:52:31.760><c> you're</c><00:52:31.920><c> going</c>"
+    },
+    {
+      "start": 3152.03,
+      "duration": 0.0,
+      "text": "smaller model cuz over time you're going"
+    },
+    {
+      "start": 3152.04,
+      "duration": 0.0,
+      "text": "smaller model cuz over time you're going to<00:52:32.440><c> you're</c><00:52:32.599><c> going</c><00:52:32.680><c> to</c><00:52:32.880><c> actually</c><00:52:33.839><c> um</c><00:52:34.720><c> spend</c>"
+    },
+    {
+      "start": 3155.069,
+      "duration": 0.0,
+      "text": "to you're going to actually um spend"
+    },
+    {
+      "start": 3155.079,
+      "duration": 0.0,
+      "text": "to you're going to actually um spend less<00:52:35.319><c> money</c><00:52:35.920><c> on</c><00:52:36.119><c> inference</c><00:52:36.520><c> of</c><00:52:36.680><c> these</c><00:52:36.839><c> models</c>"
+    },
+    {
+      "start": 3157.589,
+      "duration": 0.0,
+      "text": "less money on inference of these models"
+    },
+    {
+      "start": 3157.599,
+      "duration": 0.0,
+      "text": "less money on inference of these models so<00:52:37.880><c> 150</c><00:52:38.760><c> to</c><00:52:39.000><c> one</c><00:52:39.319><c> that's</c><00:52:39.720><c> around</c><00:52:40.240><c> what</c><00:52:40.480><c> the</c>"
+    },
+    {
+      "start": 3160.63,
+      "duration": 0.0,
+      "text": "so 150 to one that's around what the"
+    },
+    {
+      "start": 3160.64,
+      "duration": 0.0,
+      "text": "so 150 to one that's around what the best<00:52:40.920><c> models</c><00:52:41.599><c> are</c><00:52:41.880><c> trained</c><00:52:42.280><c> on</c><00:52:42.680><c> right</c><00:52:42.799><c> now</c><00:52:43.040><c> at</c>"
+    },
+    {
+      "start": 3163.109,
+      "duration": 0.0,
+      "text": "best models are trained on right now at"
+    },
+    {
+      "start": 3163.119,
+      "duration": 0.0,
+      "text": "best models are trained on right now at least<00:52:43.319><c> the</c><00:52:43.440><c> ones</c><00:52:43.680><c> that</c><00:52:43.839><c> are</c><00:52:44.760><c> that</c><00:52:44.880><c> are</c><00:52:45.040><c> used</c><00:52:45.720><c> um</c>"
+    },
+    {
+      "start": 3166.51,
+      "duration": 0.0,
+      "text": "least the ones that are that are used um"
+    },
+    {
+      "start": 3166.52,
+      "duration": 0.0,
+      "text": "least the ones that are that are used um in<00:52:46.760><c> practice</c><00:52:47.440><c> for</c><00:52:47.599><c> in</c>"
+    },
+    {
+      "start": 3168.99,
+      "duration": 0.0,
+      "text": "in practice for in"
+    },
+    {
+      "start": 3169.0,
+      "duration": 0.0,
+      "text": "in practice for in production"
+    },
+    {
+      "start": 3170.99,
+      "duration": 0.0,
+      "text": "production"
+    },
+    {
+      "start": 3171.0,
+      "duration": 0.0,
+      "text": "production great<00:52:52.000><c> any</c><00:52:52.160><c> question</c><00:52:52.400><c> on</c>"
+    },
+    {
+      "start": 3175.04,
+      "duration": 0.0,
+      "text": "chin<00:52:56.040><c> great</c><00:52:56.920><c> oh</c><00:52:57.160><c> sorry</c><00:52:57.799><c> in</c><00:52:58.079><c> practice</c><00:52:58.599><c> how</c>"
+    },
+    {
+      "start": 3178.829,
+      "duration": 0.0,
+      "text": "chin great oh sorry in practice how"
+    },
+    {
+      "start": 3178.839,
+      "duration": 0.0,
+      "text": "chin great oh sorry in practice how expensive<00:52:59.440><c> is</c><00:52:59.799><c> inference</c><00:53:00.400><c> for</c><00:53:00.599><c> these</c><00:53:00.799><c> models</c>"
+    },
+    {
+      "start": 3181.39,
+      "duration": 0.0,
+      "text": "expensive is inference for these models"
+    },
+    {
+      "start": 3181.4,
+      "duration": 0.0,
+      "text": "expensive is inference for these models rela<00:53:01.799><c> to</c>"
+    },
+    {
+      "start": 3182.51,
+      "duration": 0.0,
+      "text": "rela to"
+    },
+    {
+      "start": 3182.52,
+      "duration": 0.0,
+      "text": "rela to train<00:53:03.520><c> actually</c><00:53:03.839><c> very</c><00:53:04.160><c> expensive</c><00:53:05.160><c> uh</c><00:53:05.319><c> I</c><00:53:05.400><c> will</c>"
+    },
+    {
+      "start": 3185.63,
+      "duration": 0.0,
+      "text": "train actually very expensive uh I will"
+    },
+    {
+      "start": 3185.64,
+      "duration": 0.0,
+      "text": "train actually very expensive uh I will not<00:53:05.839><c> talk</c><00:53:06.040><c> about</c><00:53:06.240><c> inference</c><00:53:06.680><c> because</c><00:53:06.839><c> that</c>"
+    },
+    {
+      "start": 3186.95,
+      "duration": 0.0,
+      "text": "not talk about inference because that"
+    },
+    {
+      "start": 3186.96,
+      "duration": 0.0,
+      "text": "not talk about inference because that would<00:53:07.119><c> be</c><00:53:07.319><c> another</c><00:53:07.720><c> entire</c><00:53:08.520><c> lecture</c><00:53:09.119><c> but</c><00:53:09.760><c> just</c>"
+    },
+    {
+      "start": 3189.95,
+      "duration": 0.0,
+      "text": "would be another entire lecture but just"
+    },
+    {
+      "start": 3189.96,
+      "duration": 0.0,
+      "text": "would be another entire lecture but just think<00:53:10.160><c> about</c><00:53:10.520><c> Chad</c><00:53:10.799><c> GPT</c><00:53:11.400><c> where</c><00:53:11.559><c> they</c><00:53:11.799><c> have</c><00:53:12.480><c> I</c>"
+    },
+    {
+      "start": 3192.549,
+      "duration": 0.0,
+      "text": "think about Chad GPT where they have I"
+    },
+    {
+      "start": 3192.559,
+      "duration": 0.0,
+      "text": "think about Chad GPT where they have I don't<00:53:12.720><c> know</c><00:53:12.839><c> how</c><00:53:12.960><c> much</c><00:53:13.400><c> it</c><00:53:13.480><c> is</c><00:53:13.799><c> now</c><00:53:14.040><c> like</c><00:53:14.200><c> 600</c>"
+    },
+    {
+      "start": 3194.789,
+      "duration": 0.0,
+      "text": "don't know how much it is now like 600"
+    },
+    {
+      "start": 3194.799,
+      "duration": 0.0,
+      "text": "don't know how much it is now like 600 million<00:53:15.680><c> people</c><00:53:16.040><c> that</c><00:53:16.160><c> used</c><00:53:16.480><c> it</c><00:53:17.440><c> um</c><00:53:18.640><c> like</c>"
+    },
+    {
+      "start": 3199.63,
+      "duration": 0.0,
+      "text": "million people that used it um like"
+    },
+    {
+      "start": 3199.64,
+      "duration": 0.0,
+      "text": "million people that used it um like that's<00:53:20.000><c> a</c><00:53:20.319><c> lot</c>"
+    },
+    {
+      "start": 3201.829,
+      "duration": 0.0,
+      "text": "that's a lot"
+    },
+    {
+      "start": 3201.839,
+      "duration": 0.0,
+      "text": "that's a lot um<00:53:22.839><c> yeah</c><00:53:23.200><c> so</c><00:53:23.400><c> it's</c><00:53:23.599><c> actually</c><00:53:23.839><c> very</c><00:53:24.040><c> expensive</c>"
+    },
+    {
+      "start": 3204.549,
+      "duration": 0.0,
+      "text": "um yeah so it's actually very expensive"
+    },
+    {
+      "start": 3204.559,
+      "duration": 0.0,
+      "text": "um yeah so it's actually very expensive there's<00:53:24.720><c> a</c><00:53:24.880><c> lot</c><00:53:25.000><c> of</c><00:53:25.160><c> optimization</c><00:53:25.720><c> you</c><00:53:25.799><c> can</c><00:53:25.920><c> do</c>"
+    },
+    {
+      "start": 3205.99,
+      "duration": 0.0,
+      "text": "there's a lot of optimization you can do"
+    },
+    {
+      "start": 3206.0,
+      "duration": 0.0,
+      "text": "there's a lot of optimization you can do for<00:53:26.240><c> in</c><00:53:26.799><c> though</c><00:53:27.359><c> um</c><00:53:27.599><c> and</c><00:53:27.720><c> that's</c><00:53:27.880><c> an</c><00:53:28.079><c> entire</c>"
+    },
+    {
+      "start": 3208.43,
+      "duration": 0.0,
+      "text": "for in though um and that's an entire"
+    },
+    {
+      "start": 3208.44,
+      "duration": 0.0,
+      "text": "for in though um and that's an entire other<00:53:28.640><c> lecture</c><00:53:29.000><c> so</c><00:53:29.119><c> I'm</c><00:53:29.240><c> going</c><00:53:29.319><c> to</c><00:53:29.480><c> skip</c><00:53:29.799><c> that</c>"
+    },
+    {
+      "start": 3210.43,
+      "duration": 0.0,
+      "text": "other lecture so I'm going to skip that"
+    },
+    {
+      "start": 3210.44,
+      "duration": 0.0,
+      "text": "other lecture so I'm going to skip that uh<00:53:30.680><c> this</c><00:53:30.839><c> time</c><00:53:31.720><c> but</c><00:53:31.839><c> it's</c><00:53:32.000><c> very</c>"
+    },
+    {
+      "start": 3212.829,
+      "duration": 0.0,
+      "text": "uh this time but it's very"
+    },
+    {
+      "start": 3212.839,
+      "duration": 0.0,
+      "text": "uh this time but it's very interesting<00:53:33.839><c> okay</c><00:53:34.040><c> tuning</c><00:53:34.960><c> um</c><00:53:35.240><c> as</c><00:53:35.359><c> I</c><00:53:35.520><c> said</c>"
+    },
+    {
+      "start": 3215.829,
+      "duration": 0.0,
+      "text": "interesting okay tuning um as I said"
+    },
+    {
+      "start": 3215.839,
+      "duration": 0.0,
+      "text": "interesting okay tuning um as I said there<00:53:35.920><c> are</c><00:53:36.079><c> many</c><00:53:36.280><c> things</c><00:53:36.480><c> that</c><00:53:36.599><c> you</c><00:53:36.680><c> can</c><00:53:37.240><c> uh</c>"
+    },
+    {
+      "start": 3217.349,
+      "duration": 0.0,
+      "text": "there are many things that you can uh"
+    },
+    {
+      "start": 3217.359,
+      "duration": 0.0,
+      "text": "there are many things that you can uh answer<00:53:37.640><c> with</c><00:53:37.760><c> scaling</c><00:53:38.079><c> laws</c><00:53:38.400><c> I</c><00:53:38.480><c> just</c><00:53:38.640><c> try</c><00:53:38.839><c> to</c>"
+    },
+    {
+      "start": 3219.03,
+      "duration": 0.0,
+      "text": "answer with scaling laws I just try to"
+    },
+    {
+      "start": 3219.04,
+      "duration": 0.0,
+      "text": "answer with scaling laws I just try to give<00:53:39.160><c> you</c><00:53:39.720><c> two</c><00:53:40.079><c> examples</c><00:53:41.079><c> uh</c><00:53:41.200><c> but</c><00:53:41.319><c> really</c>"
+    },
+    {
+      "start": 3221.51,
+      "duration": 0.0,
+      "text": "give you two examples uh but really"
+    },
+    {
+      "start": 3221.52,
+      "duration": 0.0,
+      "text": "give you two examples uh but really there<00:53:41.640><c> are</c><00:53:41.760><c> many</c><00:53:41.960><c> things</c><00:53:42.319><c> what</c><00:53:42.559><c> data</c><00:53:42.760><c> do</c><00:53:42.880><c> you</c>"
+    },
+    {
+      "start": 3223.03,
+      "duration": 0.0,
+      "text": "there are many things what data do you"
+    },
+    {
+      "start": 3223.04,
+      "duration": 0.0,
+      "text": "there are many things what data do you use<00:53:43.400><c> what</c><00:53:43.559><c> mixture</c><00:53:44.280><c> what</c><00:53:44.640><c> data</c><00:53:44.920><c> mixing</c>"
+    },
+    {
+      "start": 3225.51,
+      "duration": 0.0,
+      "text": "use what mixture what data mixing"
+    },
+    {
+      "start": 3225.52,
+      "duration": 0.0,
+      "text": "use what mixture what data mixing waiting<00:53:46.200><c> you</c><00:53:46.359><c> use</c><00:53:46.720><c> data</c><00:53:46.920><c> mixtures</c><00:53:47.319><c> that's</c>"
+    },
+    {
+      "start": 3227.43,
+      "duration": 0.0,
+      "text": "waiting you use data mixtures that's"
+    },
+    {
+      "start": 3227.44,
+      "duration": 0.0,
+      "text": "waiting you use data mixtures that's what<00:53:47.559><c> we</c><00:53:47.680><c> talked</c><00:53:47.920><c> about</c><00:53:48.160><c> before</c><00:53:49.040><c> uh</c><00:53:49.200><c> what</c>"
+    },
+    {
+      "start": 3229.349,
+      "duration": 0.0,
+      "text": "what we talked about before uh what"
+    },
+    {
+      "start": 3229.359,
+      "duration": 0.0,
+      "text": "what we talked about before uh what architecture<00:53:49.839><c> you</c><00:53:50.000><c> use</c><00:53:50.599><c> whether</c><00:53:50.799><c> you</c><00:53:50.880><c> should</c>"
+    },
+    {
+      "start": 3231.069,
+      "duration": 0.0,
+      "text": "architecture you use whether you should"
+    },
+    {
+      "start": 3231.079,
+      "duration": 0.0,
+      "text": "architecture you use whether you should make<00:53:51.240><c> your</c><00:53:51.359><c> models</c><00:53:52.079><c> uh</c><00:53:52.280><c> wider</c><00:53:52.720><c> or</c><00:53:53.000><c> deeper</c><00:53:54.000><c> um</c>"
+    },
+    {
+      "start": 3234.19,
+      "duration": 0.0,
+      "text": "make your models uh wider or deeper um"
+    },
+    {
+      "start": 3234.2,
+      "duration": 0.0,
+      "text": "make your models uh wider or deeper um should<00:53:54.400><c> you</c><00:53:54.559><c> be</c><00:53:55.200><c> paying</c><00:53:55.440><c> for</c><00:53:55.599><c> more</c><00:53:55.799><c> gpus</c><00:53:56.240><c> or</c>"
+    },
+    {
+      "start": 3236.51,
+      "duration": 0.0,
+      "text": "should you be paying for more gpus or"
+    },
+    {
+      "start": 3236.52,
+      "duration": 0.0,
+      "text": "should you be paying for more gpus or actually<00:53:56.720><c> collecting</c><00:53:57.079><c> more</c><00:53:57.319><c> data</c><00:53:58.280><c> um</c><00:53:59.000><c> all</c>"
+    },
+    {
+      "start": 3239.19,
+      "duration": 0.0,
+      "text": "actually collecting more data um all"
+    },
+    {
+      "start": 3239.2,
+      "duration": 0.0,
+      "text": "actually collecting more data um all these<00:53:59.359><c> things</c><00:53:59.599><c> are</c><00:53:59.799><c> things</c><00:54:00.000><c> you</c><00:54:00.119><c> can</c><00:54:00.240><c> try</c><00:54:00.440><c> to</c>"
+    },
+    {
+      "start": 3240.589,
+      "duration": 0.0,
+      "text": "these things are things you can try to"
+    },
+    {
+      "start": 3240.599,
+      "duration": 0.0,
+      "text": "these things are things you can try to answer<00:54:00.839><c> with</c><00:54:00.960><c> scaling</c>"
+    },
+    {
+      "start": 3242.39,
+      "duration": 0.0,
+      "text": "answer with scaling"
+    },
+    {
+      "start": 3242.4,
+      "duration": 0.0,
+      "text": "answer with scaling laws<00:54:03.400><c> one</c><00:54:03.640><c> thing</c><00:54:03.799><c> I</c><00:54:03.880><c> want</c><00:54:04.000><c> to</c><00:54:04.160><c> say</c><00:54:04.520><c> is</c><00:54:04.680><c> the</c><00:54:04.799><c> bit</c>"
+    },
+    {
+      "start": 3245.15,
+      "duration": 0.0,
+      "text": "laws one thing I want to say is the bit"
+    },
+    {
+      "start": 3245.16,
+      "duration": 0.0,
+      "text": "laws one thing I want to say is the bit lesson<00:54:05.559><c> if</c><00:54:05.640><c> you</c><00:54:05.760><c> ever</c><00:54:06.040><c> heard</c><00:54:06.720><c> of</c><00:54:06.960><c> Richard</c>"
+    },
+    {
+      "start": 3247.309,
+      "duration": 0.0,
+      "text": "lesson if you ever heard of Richard"
+    },
+    {
+      "start": 3247.319,
+      "duration": 0.0,
+      "text": "lesson if you ever heard of Richard sudden<00:54:08.160><c> a</c><00:54:08.359><c> very</c><00:54:08.599><c> famous</c><00:54:08.920><c> blog</c><00:54:09.200><c> post</c><00:54:09.400><c> in</c><00:54:09.880><c> 2019</c>"
+    },
+    {
+      "start": 3250.87,
+      "duration": 0.0,
+      "text": "sudden a very famous blog post in 2019"
+    },
+    {
+      "start": 3250.88,
+      "duration": 0.0,
+      "text": "sudden a very famous blog post in 2019 um<00:54:11.400><c> what</c><00:54:11.559><c> he</c><00:54:11.920><c> realized</c><00:54:12.920><c> uh</c><00:54:13.640><c> which</c><00:54:14.640><c> I</c><00:54:14.799><c> think</c><00:54:15.520><c> not</c>"
+    },
+    {
+      "start": 3255.75,
+      "duration": 0.0,
+      "text": "um what he realized uh which I think not"
+    },
+    {
+      "start": 3255.76,
+      "duration": 0.0,
+      "text": "um what he realized uh which I think not enough<00:54:16.000><c> people</c><00:54:16.200><c> realize</c><00:54:16.599><c> I</c><00:54:16.760><c> didn't</c>"
+    },
+    {
+      "start": 3257.069,
+      "duration": 0.0,
+      "text": "enough people realize I didn't"
+    },
+    {
+      "start": 3257.079,
+      "duration": 0.0,
+      "text": "enough people realize I didn't definitely<00:54:17.480><c> did</c><00:54:17.640><c> not</c><00:54:17.799><c> realize</c><00:54:18.160><c> at</c><00:54:18.359><c> that</c><00:54:18.520><c> time</c>"
+    },
+    {
+      "start": 3259.349,
+      "duration": 0.0,
+      "text": "definitely did not realize at that time"
+    },
+    {
+      "start": 3259.359,
+      "duration": 0.0,
+      "text": "definitely did not realize at that time um<00:54:20.040><c> is</c><00:54:20.160><c> that</c><00:54:20.680><c> once</c><00:54:20.839><c> you</c><00:54:20.960><c> see</c><00:54:21.240><c> these</c><00:54:21.400><c> type</c><00:54:21.559><c> of</c>"
+    },
+    {
+      "start": 3261.63,
+      "duration": 0.0,
+      "text": "um is that once you see these type of"
+    },
+    {
+      "start": 3261.64,
+      "duration": 0.0,
+      "text": "um is that once you see these type of scaling<00:54:21.960><c> laws</c><00:54:22.440><c> you</c><00:54:22.599><c> know</c><00:54:22.839><c> that</c><00:54:23.000><c> the</c><00:54:23.119><c> more</c>"
+    },
+    {
+      "start": 3263.309,
+      "duration": 0.0,
+      "text": "scaling laws you know that the more"
+    },
+    {
+      "start": 3263.319,
+      "duration": 0.0,
+      "text": "scaling laws you know that the more compute<00:54:23.720><c> you</c><00:54:23.920><c> have</c><00:54:24.319><c> the</c><00:54:24.440><c> better</c><00:54:24.720><c> models</c><00:54:25.440><c> you</c>"
+    },
+    {
+      "start": 3265.549,
+      "duration": 0.0,
+      "text": "compute you have the better models you"
+    },
+    {
+      "start": 3265.559,
+      "duration": 0.0,
+      "text": "compute you have the better models you will<00:54:25.799><c> get</c><00:54:26.119><c> so</c><00:54:26.480><c> with</c><00:54:26.599><c> skill</c><00:54:26.839><c> you</c><00:54:26.920><c> will</c><00:54:27.079><c> get</c>"
+    },
+    {
+      "start": 3267.23,
+      "duration": 0.0,
+      "text": "will get so with skill you will get"
+    },
+    {
+      "start": 3267.24,
+      "duration": 0.0,
+      "text": "will get so with skill you will get better<00:54:27.480><c> model</c><00:54:28.119><c> and</c><00:54:28.200><c> you</c><00:54:28.359><c> also</c><00:54:28.599><c> know</c><00:54:28.839><c> by</c><00:54:29.119><c> Mo</c><00:54:29.559><c> law</c>"
+    },
+    {
+      "start": 3270.109,
+      "duration": 0.0,
+      "text": "better model and you also know by Mo law"
+    },
+    {
+      "start": 3270.119,
+      "duration": 0.0,
+      "text": "better model and you also know by Mo law or<00:54:30.359><c> these</c><00:54:30.559><c> type</c><00:54:30.760><c> of</c><00:54:30.960><c> variant</c><00:54:31.280><c> of</c><00:54:31.440><c> Mo</c><00:54:31.760><c> law</c><00:54:32.200><c> that</c>"
+    },
+    {
+      "start": 3272.349,
+      "duration": 0.0,
+      "text": "or these type of variant of Mo law that"
+    },
+    {
+      "start": 3272.359,
+      "duration": 0.0,
+      "text": "or these type of variant of Mo law that you<00:54:32.440><c> will</c><00:54:32.720><c> always</c><00:54:33.000><c> have</c><00:54:33.200><c> better</c><00:54:33.440><c> compute</c><00:54:34.079><c> then</c>"
+    },
+    {
+      "start": 3274.23,
+      "duration": 0.0,
+      "text": "you will always have better compute then"
+    },
+    {
+      "start": 3274.24,
+      "duration": 0.0,
+      "text": "you will always have better compute then the<00:54:34.480><c> only</c><00:54:34.799><c> thing</c><00:54:35.400><c> that</c><00:54:35.640><c> matters</c><00:54:36.400><c> is</c><00:54:36.599><c> just</c><00:54:36.760><c> to</c>"
+    },
+    {
+      "start": 3277.069,
+      "duration": 0.0,
+      "text": "the only thing that matters is just to"
+    },
+    {
+      "start": 3277.079,
+      "duration": 0.0,
+      "text": "the only thing that matters is just to have<00:54:37.359><c> architectures</c><00:54:38.079><c> that</c><00:54:38.200><c> can</c><00:54:38.400><c> leverage</c>"
+    },
+    {
+      "start": 3279.109,
+      "duration": 0.0,
+      "text": "have architectures that can leverage"
+    },
+    {
+      "start": 3279.119,
+      "duration": 0.0,
+      "text": "have architectures that can leverage computation<00:54:39.920><c> so</c><00:54:40.160><c> what</c><00:54:40.319><c> matters</c><00:54:41.119><c> is</c><00:54:41.319><c> basically</c>"
+    },
+    {
+      "start": 3281.789,
+      "duration": 0.0,
+      "text": "computation so what matters is basically"
+    },
+    {
+      "start": 3281.799,
+      "duration": 0.0,
+      "text": "computation so what matters is basically systems<00:54:42.799><c> data</c><00:54:43.559><c> and</c><00:54:43.760><c> less</c><00:54:44.000><c> so</c><00:54:44.280><c> the</c>"
+    },
+    {
+      "start": 3284.43,
+      "duration": 0.0,
+      "text": "systems data and less so the"
+    },
+    {
+      "start": 3284.44,
+      "duration": 0.0,
+      "text": "systems data and less so the architecture<00:54:45.079><c> like</c><00:54:45.200><c> the</c><00:54:45.319><c> small</c><00:54:45.640><c> architecture</c>"
+    },
+    {
+      "start": 3286.15,
+      "duration": 0.0,
+      "text": "architecture like the small architecture"
+    },
+    {
+      "start": 3286.16,
+      "duration": 0.0,
+      "text": "architecture like the small architecture differences<00:54:46.640><c> like</c><00:54:46.880><c> your</c><00:54:47.280><c> your</c><00:54:47.680><c> your</c>"
+    },
+    {
+      "start": 3287.87,
+      "duration": 0.0,
+      "text": "differences like your your your"
+    },
+    {
+      "start": 3287.88,
+      "duration": 0.0,
+      "text": "differences like your your your activation<00:54:48.319><c> and</c><00:54:48.480><c> things</c><00:54:48.680><c> like</c><00:54:48.880><c> this</c><00:54:49.640><c> uh</c><00:54:49.799><c> so</c><00:54:49.960><c> I</c>"
+    },
+    {
+      "start": 3290.03,
+      "duration": 0.0,
+      "text": "activation and things like this uh so I"
+    },
+    {
+      "start": 3290.04,
+      "duration": 0.0,
+      "text": "activation and things like this uh so I think<00:54:50.200><c> that's</c><00:54:50.400><c> like</c><00:54:50.559><c> one</c><00:54:50.680><c> of</c><00:54:50.799><c> the</c><00:54:50.920><c> reasons</c><00:54:51.280><c> why</c>"
+    },
+    {
+      "start": 3291.47,
+      "duration": 0.0,
+      "text": "think that's like one of the reasons why"
+    },
+    {
+      "start": 3291.48,
+      "duration": 0.0,
+      "text": "think that's like one of the reasons why most<00:54:51.640><c> of</c><00:54:51.839><c> research</c><00:54:52.200><c> focuses</c><00:54:53.040><c> on</c><00:54:53.559><c> um</c><00:54:54.440><c> some</c>"
+    },
+    {
+      "start": 3294.67,
+      "duration": 0.0,
+      "text": "most of research focuses on um some"
+    },
+    {
+      "start": 3294.68,
+      "duration": 0.0,
+      "text": "most of research focuses on um some things<00:54:54.960><c> that</c><00:54:55.119><c> for</c><00:54:55.400><c> industry</c><00:54:55.720><c> matters</c><00:54:56.079><c> less</c>"
+    },
+    {
+      "start": 3296.829,
+      "duration": 0.0,
+      "text": "things that for industry matters less"
+    },
+    {
+      "start": 3296.839,
+      "duration": 0.0,
+      "text": "things that for industry matters less and<00:54:56.960><c> I</c><00:54:57.119><c> was</c><00:54:57.280><c> one</c><00:54:57.400><c> of</c><00:54:57.559><c> those</c><00:54:57.760><c> researchers</c><00:54:58.280><c> for</c><00:54:58.799><c> a</c>"
+    },
+    {
+      "start": 3299.39,
+      "duration": 0.0,
+      "text": "and I was one of those researchers for a"
+    },
+    {
+      "start": 3299.4,
+      "duration": 0.0,
+      "text": "and I was one of those researchers for a large<00:54:59.680><c> part</c><00:54:59.839><c> of</c><00:55:00.040><c> my</c><00:55:00.319><c> my</c><00:55:00.880><c> career</c><00:55:01.880><c> um</c><00:55:02.520><c> so</c><00:55:02.720><c> don't</c>"
+    },
+    {
+      "start": 3302.91,
+      "duration": 0.0,
+      "text": "large part of my my career um so don't"
+    },
+    {
+      "start": 3302.92,
+      "duration": 0.0,
+      "text": "large part of my my career um so don't spend<00:55:03.200><c> time</c><00:55:03.480><c> over</c><00:55:03.880><c> complicating</c><00:55:04.880><c> do</c><00:55:05.200><c> the</c>"
+    },
+    {
+      "start": 3305.39,
+      "duration": 0.0,
+      "text": "spend time over complicating do the"
+    },
+    {
+      "start": 3305.4,
+      "duration": 0.0,
+      "text": "spend time over complicating do the simple<00:55:05.720><c> things</c><00:55:06.280><c> do</c><00:55:06.440><c> it</c><00:55:06.640><c> well</c><00:55:07.040><c> seal</c><00:55:07.559><c> them</c>"
+    },
+    {
+      "start": 3308.109,
+      "duration": 0.0,
+      "text": "simple things do it well seal them"
+    },
+    {
+      "start": 3308.119,
+      "duration": 0.0,
+      "text": "simple things do it well seal them that's<00:55:08.359><c> really</c><00:55:08.640><c> what</c><00:55:08.920><c> openi</c><00:55:09.520><c> taught</c><00:55:09.760><c> us</c><00:55:10.359><c> with</c>"
+    },
+    {
+      "start": 3310.75,
+      "duration": 0.0,
+      "text": "that's really what openi taught us with"
+    },
+    {
+      "start": 3310.76,
+      "duration": 0.0,
+      "text": "that's really what openi taught us with um<00:55:11.119><c> with</c><00:55:11.280><c> chat</c><00:55:11.480><c> gpg</c><00:55:12.079><c> and</c><00:55:12.240><c> with</c><00:55:12.440><c> all</c><00:55:12.559><c> the</c><00:55:12.680><c> gpts</c>"
+    },
+    {
+      "start": 3314.47,
+      "duration": 0.0,
+      "text": "um with chat gpg and with all the gpts"
+    },
+    {
+      "start": 3314.48,
+      "duration": 0.0,
+      "text": "um with chat gpg and with all the gpts before<00:55:15.480><c> okay</c><00:55:15.640><c> I</c><00:55:15.720><c> want</c><00:55:15.839><c> to</c><00:55:16.000><c> give</c><00:55:16.119><c> you</c><00:55:16.319><c> some</c>"
+    },
+    {
+      "start": 3316.67,
+      "duration": 0.0,
+      "text": "before okay I want to give you some"
+    },
+    {
+      "start": 3316.68,
+      "duration": 0.0,
+      "text": "before okay I want to give you some backup<00:55:17.000><c> the</c><00:55:17.200><c> envelope</c><00:55:18.200><c> computation</c><00:55:18.799><c> so</c><00:55:18.960><c> I</c>"
+    },
+    {
+      "start": 3319.03,
+      "duration": 0.0,
+      "text": "backup the envelope computation so I"
+    },
+    {
+      "start": 3319.04,
+      "duration": 0.0,
+      "text": "backup the envelope computation so I might<00:55:19.200><c> be</c><00:55:19.400><c> off</c><00:55:19.720><c> by</c><00:55:19.839><c> a</c><00:55:19.960><c> few</c><00:55:20.119><c> factors</c><00:55:20.559><c> here</c><00:55:20.720><c> but</c><00:55:20.839><c> I</c>"
+    },
+    {
+      "start": 3320.91,
+      "duration": 0.0,
+      "text": "might be off by a few factors here but I"
+    },
+    {
+      "start": 3320.92,
+      "duration": 0.0,
+      "text": "might be off by a few factors here but I just<00:55:21.000><c> want</c><00:55:21.119><c> to</c><00:55:21.280><c> give</c><00:55:21.400><c> you</c><00:55:21.559><c> a</c><00:55:21.760><c> sense</c><00:55:22.079><c> of</c><00:55:22.319><c> how</c>"
+    },
+    {
+      "start": 3322.63,
+      "duration": 0.0,
+      "text": "just want to give you a sense of how"
+    },
+    {
+      "start": 3322.64,
+      "duration": 0.0,
+      "text": "just want to give you a sense of how costly<00:55:23.319><c> it</c><00:55:23.440><c> is</c><00:55:23.559><c> to</c><00:55:23.680><c> train</c><00:55:23.960><c> some</c><00:55:24.079><c> of</c><00:55:24.200><c> these</c>"
+    },
+    {
+      "start": 3324.349,
+      "duration": 0.0,
+      "text": "costly it is to train some of these"
+    },
+    {
+      "start": 3324.359,
+      "duration": 0.0,
+      "text": "costly it is to train some of these models<00:55:25.280><c> I'll</c><00:55:25.480><c> give</c><00:55:25.640><c> as</c><00:55:25.760><c> an</c><00:55:25.920><c> example</c>"
+    },
+    {
+      "start": 3326.829,
+      "duration": 0.0,
+      "text": "models I'll give as an example"
+    },
+    {
+      "start": 3326.839,
+      "duration": 0.0,
+      "text": "models I'll give as an example Lama<00:55:27.240><c> 3</c><00:55:27.520><c> 400b</c><00:55:28.280><c> which</c><00:55:28.359><c> is</c><00:55:28.559><c> currently</c><00:55:28.960><c> the</c><00:55:29.079><c> best</c>"
+    },
+    {
+      "start": 3329.309,
+      "duration": 0.0,
+      "text": "Lama 3 400b which is currently the best"
+    },
+    {
+      "start": 3329.319,
+      "duration": 0.0,
+      "text": "Lama 3 400b which is currently the best open<00:55:29.559><c> source</c><00:55:29.839><c> model</c><00:55:30.119><c> that</c><00:55:30.240><c> you</c><00:55:30.319><c> can</c><00:55:30.640><c> get</c><00:55:31.640><c> uh</c><00:55:31.880><c> it</c>"
+    },
+    {
+      "start": 3332.029,
+      "duration": 0.0,
+      "text": "open source model that you can get uh it"
+    },
+    {
+      "start": 3332.039,
+      "duration": 0.0,
+      "text": "open source model that you can get uh it was<00:55:32.240><c> trained</c><00:55:32.680><c> on</c><00:55:33.240><c> 15.6</c><00:55:34.079><c> tokens</c><00:55:35.039><c> it</c><00:55:35.240><c> has</c><00:55:35.760><c> 45</c>"
+    },
+    {
+      "start": 3336.75,
+      "duration": 0.0,
+      "text": "was trained on 15.6 tokens it has 45"
+    },
+    {
+      "start": 3336.76,
+      "duration": 0.0,
+      "text": "was trained on 15.6 tokens it has 45 billion<00:55:37.119><c> parameters</c><00:55:37.839><c> so</c><00:55:38.160><c> just</c><00:55:38.440><c> now</c><00:55:38.599><c> that</c><00:55:38.720><c> you</c>"
+    },
+    {
+      "start": 3338.87,
+      "duration": 0.0,
+      "text": "billion parameters so just now that you"
+    },
+    {
+      "start": 3338.88,
+      "duration": 0.0,
+      "text": "billion parameters so just now that you know<00:55:39.119><c> what</c><00:55:39.240><c> is</c><00:55:39.440><c> like</c><00:55:39.680><c> this</c><00:55:40.119><c> uh</c><00:55:40.559><c> optimal</c><00:55:41.119><c> tokens</c>"
+    },
+    {
+      "start": 3341.51,
+      "duration": 0.0,
+      "text": "know what is like this uh optimal tokens"
+    },
+    {
+      "start": 3341.52,
+      "duration": 0.0,
+      "text": "know what is like this uh optimal tokens per<00:55:41.680><c> parameter</c><00:55:42.160><c> that's</c><00:55:42.400><c> around</c><00:55:42.599><c> 40</c><00:55:43.200><c> so</c><00:55:43.400><c> that's</c>"
+    },
+    {
+      "start": 3343.91,
+      "duration": 0.0,
+      "text": "per parameter that's around 40 so that's"
+    },
+    {
+      "start": 3343.92,
+      "duration": 0.0,
+      "text": "per parameter that's around 40 so that's a<00:55:44.000><c> little</c><00:55:44.160><c> bit</c><00:55:44.319><c> more</c><00:55:44.480><c> than</c><00:55:44.640><c> chinchilla</c><00:55:45.480><c> but</c>"
+    },
+    {
+      "start": 3345.71,
+      "duration": 0.0,
+      "text": "a little bit more than chinchilla but"
+    },
+    {
+      "start": 3345.72,
+      "duration": 0.0,
+      "text": "a little bit more than chinchilla but less<00:55:45.960><c> than</c><00:55:46.160><c> this</c><00:55:46.400><c> like</c><00:55:46.720><c> inference</c><00:55:47.720><c> uh</c><00:55:48.039><c> optimal</c>"
+    },
+    {
+      "start": 3349.029,
+      "duration": 0.0,
+      "text": "less than this like inference uh optimal"
+    },
+    {
+      "start": 3349.039,
+      "duration": 0.0,
+      "text": "less than this like inference uh optimal um<00:55:50.039><c> model</c><00:55:50.440><c> so</c><00:55:50.599><c> they</c><00:55:50.720><c> went</c><00:55:50.960><c> for</c><00:55:51.119><c> training</c>"
+    },
+    {
+      "start": 3352.39,
+      "duration": 0.0,
+      "text": "um model so they went for training"
+    },
+    {
+      "start": 3352.4,
+      "duration": 0.0,
+      "text": "um model so they went for training optimality<00:55:53.400><c> uh</c><00:55:53.520><c> flops</c><00:55:54.119><c> for</c><00:55:54.359><c> this</c><00:55:54.559><c> model</c><00:55:55.000><c> so</c>"
+    },
+    {
+      "start": 3355.39,
+      "duration": 0.0,
+      "text": "optimality uh flops for this model so"
+    },
+    {
+      "start": 3355.4,
+      "duration": 0.0,
+      "text": "optimality uh flops for this model so one<00:55:55.680><c> simple</c><00:55:56.559><c> uh</c><00:55:56.680><c> way</c><00:55:56.839><c> to</c><00:55:57.000><c> compute</c><00:55:57.319><c> flops</c><00:55:57.720><c> is</c>"
+    },
+    {
+      "start": 3357.95,
+      "duration": 0.0,
+      "text": "one simple uh way to compute flops is"
+    },
+    {
+      "start": 3357.96,
+      "duration": 0.0,
+      "text": "one simple uh way to compute flops is six<00:55:58.960><c> uh</c><00:55:59.240><c> times</c><00:55:59.720><c> the</c><00:55:59.839><c> number</c><00:56:00.079><c> of</c><00:56:00.240><c> parameters</c>"
+    },
+    {
+      "start": 3360.829,
+      "duration": 0.0,
+      "text": "six uh times the number of parameters"
+    },
+    {
+      "start": 3360.839,
+      "duration": 0.0,
+      "text": "six uh times the number of parameters times<00:56:01.160><c> the</c><00:56:01.240><c> number</c><00:56:01.440><c> of</c><00:56:01.520><c> data</c><00:56:01.839><c> you</c><00:56:01.960><c> train</c><00:56:02.200><c> on</c><00:56:02.880><c> uh</c>"
+    },
+    {
+      "start": 3362.99,
+      "duration": 0.0,
+      "text": "times the number of data you train on uh"
+    },
+    {
+      "start": 3363.0,
+      "duration": 0.0,
+      "text": "times the number of data you train on uh so<00:56:03.119><c> if</c><00:56:03.200><c> you</c><00:56:03.280><c> do</c><00:56:03.400><c> the</c><00:56:03.520><c> simple</c><00:56:03.839><c> calculation</c><00:56:04.440><c> here</c>"
+    },
+    {
+      "start": 3364.789,
+      "duration": 0.0,
+      "text": "so if you do the simple calculation here"
+    },
+    {
+      "start": 3364.799,
+      "duration": 0.0,
+      "text": "so if you do the simple calculation here it's<00:56:05.000><c> 3.8</c><00:56:05.799><c> e25</c><00:56:06.720><c> flops</c><00:56:07.720><c> the</c><00:56:07.839><c> reason</c><00:56:08.160><c> why</c><00:56:08.319><c> this</c>"
+    },
+    {
+      "start": 3368.39,
+      "duration": 0.0,
+      "text": "it's 3.8 e25 flops the reason why this"
+    },
+    {
+      "start": 3368.4,
+      "duration": 0.0,
+      "text": "it's 3.8 e25 flops the reason why this is<00:56:08.599><c> important</c><00:56:09.119><c> is</c><00:56:09.240><c> that</c><00:56:09.440><c> if</c><00:56:09.559><c> you</c><00:56:09.680><c> follow</c><00:56:10.039><c> the</c>"
+    },
+    {
+      "start": 3370.109,
+      "duration": 0.0,
+      "text": "is important is that if you follow the"
+    },
+    {
+      "start": 3370.119,
+      "duration": 0.0,
+      "text": "is important is that if you follow the little<00:56:10.280><c> bit</c><00:56:10.440><c> the</c><00:56:10.520><c> news</c><00:56:10.760><c> there's</c><00:56:10.920><c> an</c><00:56:11.079><c> executive</c>"
+    },
+    {
+      "start": 3371.589,
+      "duration": 0.0,
+      "text": "little bit the news there's an executive"
+    },
+    {
+      "start": 3371.599,
+      "duration": 0.0,
+      "text": "little bit the news there's an executive order<00:56:12.319><c> from</c><00:56:12.520><c> Biden</c><00:56:12.920><c> that</c><00:56:13.119><c> basically</c><00:56:13.440><c> says</c>"
+    },
+    {
+      "start": 3373.63,
+      "duration": 0.0,
+      "text": "order from Biden that basically says"
+    },
+    {
+      "start": 3373.64,
+      "duration": 0.0,
+      "text": "order from Biden that basically says that<00:56:13.799><c> once</c><00:56:13.960><c> you</c><00:56:14.079><c> have</c><00:56:14.880><c> uh</c><00:56:15.000><c> 1</c><00:56:15.799><c> e26</c><00:56:16.799><c> parameters</c>"
+    },
+    {
+      "start": 3377.789,
+      "duration": 0.0,
+      "text": "that once you have uh 1 e26 parameters"
+    },
+    {
+      "start": 3377.799,
+      "duration": 0.0,
+      "text": "that once you have uh 1 e26 parameters uh<00:56:17.920><c> sorry</c><00:56:18.240><c> flops</c><00:56:19.240><c> uh</c><00:56:19.359><c> then</c><00:56:19.520><c> you</c><00:56:19.640><c> have</c><00:56:19.799><c> special</c>"
+    },
+    {
+      "start": 3380.069,
+      "duration": 0.0,
+      "text": "uh sorry flops uh then you have special"
+    },
+    {
+      "start": 3380.079,
+      "duration": 0.0,
+      "text": "uh sorry flops uh then you have special scrutiny<00:56:20.559><c> on</c><00:56:20.680><c> your</c><00:56:20.799><c> models</c><00:56:21.359><c> so</c><00:56:21.599><c> they</c><00:56:21.760><c> went</c><00:56:22.319><c> 2x</c>"
+    },
+    {
+      "start": 3382.95,
+      "duration": 0.0,
+      "text": "scrutiny on your models so they went 2x"
+    },
+    {
+      "start": 3382.96,
+      "duration": 0.0,
+      "text": "scrutiny on your models so they went 2x less<00:56:23.160><c> than</c><00:56:23.359><c> that</c><00:56:23.520><c> so</c><00:56:23.720><c> they</c><00:56:23.920><c> really</c><00:56:24.119><c> went</c><00:56:24.440><c> right</c>"
+    },
+    {
+      "start": 3384.63,
+      "duration": 0.0,
+      "text": "less than that so they really went right"
+    },
+    {
+      "start": 3384.64,
+      "duration": 0.0,
+      "text": "less than that so they really went right below<00:56:25.000><c> this</c><00:56:25.480><c> to</c><00:56:25.640><c> not</c><00:56:25.839><c> have</c><00:56:25.960><c> special</c><00:56:26.440><c> scrutiny</c>"
+    },
+    {
+      "start": 3387.27,
+      "duration": 0.0,
+      "text": "below this to not have special scrutiny"
+    },
+    {
+      "start": 3387.28,
+      "duration": 0.0,
+      "text": "below this to not have special scrutiny so<00:56:27.559><c> 38</c><00:56:28.559><c> uh</c><00:56:28.680><c> I</c><00:56:28.799><c> might</c><00:56:28.960><c> be</c><00:56:29.119><c> off</c><00:56:29.319><c> by</c><00:56:29.480><c> a</c><00:56:29.599><c> little</c><00:56:29.760><c> bit</c>"
+    },
+    {
+      "start": 3389.91,
+      "duration": 0.0,
+      "text": "so 38 uh I might be off by a little bit"
+    },
+    {
+      "start": 3389.92,
+      "duration": 0.0,
+      "text": "so 38 uh I might be off by a little bit but<00:56:30.039><c> it's</c><00:56:30.200><c> definitely</c><00:56:30.680><c> under</c><00:56:31.000><c> the</c><00:56:31.440><c> 1</c>"
+    },
+    {
+      "start": 3394.52,
+      "duration": 0.0,
+      "text": "26<00:56:35.520><c> oh</c><00:56:36.079><c> um</c><00:56:36.640><c> so</c><00:56:37.200><c> paramet</c><00:56:37.720><c> p</c><00:56:37.920><c> is</c><00:56:38.079><c> parameters</c><00:56:39.000><c> n</c><00:56:39.720><c> is</c>"
+    },
+    {
+      "start": 3399.99,
+      "duration": 0.0,
+      "text": "26 oh um so paramet p is parameters n is"
+    },
+    {
+      "start": 3400.0,
+      "duration": 0.0,
+      "text": "26 oh um so paramet p is parameters n is data<00:56:40.559><c> number</c><00:56:40.799><c> of</c><00:56:40.960><c> tokens</c><00:56:41.880><c> this</c><00:56:42.039><c> is</c><00:56:42.400><c> a</c><00:56:43.280><c> uh</c><00:56:43.599><c> this</c>"
+    },
+    {
+      "start": 3403.67,
+      "duration": 0.0,
+      "text": "data number of tokens this is a uh this"
+    },
+    {
+      "start": 3403.68,
+      "duration": 0.0,
+      "text": "data number of tokens this is a uh this is<00:56:43.799><c> just</c><00:56:43.920><c> an</c>"
+    },
+    {
+      "start": 3404.91,
+      "duration": 0.0,
+      "text": "is just an"
+    },
+    {
+      "start": 3404.92,
+      "duration": 0.0,
+      "text": "is just an approximation<00:56:45.920><c> we</c>"
+    },
+    {
+      "start": 3407.27,
+      "duration": 0.0,
+      "text": "approximation we"
+    },
+    {
+      "start": 3407.28,
+      "duration": 0.0,
+      "text": "approximation we yeah<00:56:48.280><c> okay</c><00:56:48.880><c> uh</c><00:56:49.079><c> compute</c><00:56:49.960><c> and</c><00:56:50.480><c> we</c><00:56:50.599><c> know</c><00:56:50.880><c> that</c>"
+    },
+    {
+      "start": 3410.99,
+      "duration": 0.0,
+      "text": "yeah okay uh compute and we know that"
+    },
+    {
+      "start": 3411.0,
+      "duration": 0.0,
+      "text": "yeah okay uh compute and we know that they<00:56:51.160><c> trained</c><00:56:51.520><c> on</c><00:56:51.799><c> 16,000</c>"
+    },
+    {
+      "start": 3413.109,
+      "duration": 0.0,
+      "text": "they trained on 16,000"
+    },
+    {
+      "start": 3413.119,
+      "duration": 0.0,
+      "text": "they trained on 16,000 h100s<00:56:54.319><c> um</c><00:56:55.319><c> and</c><00:56:55.480><c> we</c><00:56:55.599><c> know</c><00:56:55.720><c> the</c><00:56:55.839><c> throughput</c><00:56:56.280><c> but</c>"
+    },
+    {
+      "start": 3416.67,
+      "duration": 0.0,
+      "text": "h100s um and we know the throughput but"
+    },
+    {
+      "start": 3416.68,
+      "duration": 0.0,
+      "text": "h100s um and we know the throughput but they<00:56:56.880><c> they</c><00:56:56.960><c> said</c><00:56:57.200><c> it</c><00:56:57.400><c> too</c><00:56:58.400><c> uh</c><00:56:58.520><c> so</c><00:56:58.760><c> if</c><00:56:58.880><c> you</c><00:56:58.960><c> do</c>"
+    },
+    {
+      "start": 3419.109,
+      "duration": 0.0,
+      "text": "they they said it too uh so if you do"
+    },
+    {
+      "start": 3419.119,
+      "duration": 0.0,
+      "text": "they they said it too uh so if you do the<00:56:59.280><c> computation</c><00:57:00.200><c> it</c><00:57:00.359><c> takes</c><00:57:00.640><c> around</c><00:57:00.960><c> 70</c><00:57:01.480><c> days</c>"
+    },
+    {
+      "start": 3422.349,
+      "duration": 0.0,
+      "text": "the computation it takes around 70 days"
+    },
+    {
+      "start": 3422.359,
+      "duration": 0.0,
+      "text": "the computation it takes around 70 days um<00:57:02.839><c> or</c><00:57:03.079><c> 26</c><00:57:03.640><c> million</c><00:57:04.039><c> GPU</c><00:57:04.640><c> hours</c><00:57:05.480><c> at</c><00:57:05.599><c> least</c>"
+    },
+    {
+      "start": 3425.75,
+      "duration": 0.0,
+      "text": "um or 26 million GPU hours at least"
+    },
+    {
+      "start": 3425.76,
+      "duration": 0.0,
+      "text": "um or 26 million GPU hours at least that's<00:57:05.960><c> with</c><00:57:06.200><c> my</c><00:57:06.839><c> uh</c><00:57:07.039><c> back</c><00:57:07.200><c> of</c><00:57:07.359><c> the</c><00:57:07.480><c> envelope</c>"
+    },
+    {
+      "start": 3427.91,
+      "duration": 0.0,
+      "text": "that's with my uh back of the envelope"
+    },
+    {
+      "start": 3427.92,
+      "duration": 0.0,
+      "text": "that's with my uh back of the envelope computation<00:57:08.480><c> they</c><00:57:08.640><c> actually</c><00:57:08.839><c> said</c><00:57:09.079><c> that</c><00:57:09.200><c> they</c>"
+    },
+    {
+      "start": 3429.309,
+      "duration": 0.0,
+      "text": "computation they actually said that they"
+    },
+    {
+      "start": 3429.319,
+      "duration": 0.0,
+      "text": "computation they actually said that they use<00:57:09.799><c> 30</c><00:57:10.200><c> million</c><00:57:10.680><c> instead</c><00:57:10.920><c> of</c><00:57:11.319><c> 26</c><00:57:11.760><c> million</c><00:57:12.079><c> GPU</c>"
+    },
+    {
+      "start": 3432.63,
+      "duration": 0.0,
+      "text": "use 30 million instead of 26 million GPU"
+    },
+    {
+      "start": 3432.64,
+      "duration": 0.0,
+      "text": "use 30 million instead of 26 million GPU hours<00:57:13.559><c> um</c><00:57:14.000><c> so</c><00:57:14.240><c> maybe</c><00:57:14.480><c> they</c><00:57:14.640><c> had</c><00:57:14.880><c> like</c><00:57:15.520><c> some</c><00:57:16.520><c> uh</c>"
+    },
+    {
+      "start": 3436.63,
+      "duration": 0.0,
+      "text": "hours um so maybe they had like some uh"
+    },
+    {
+      "start": 3436.64,
+      "duration": 0.0,
+      "text": "hours um so maybe they had like some uh some<00:57:16.880><c> challenges</c><00:57:17.599><c> I</c><00:57:17.680><c> don't</c><00:57:17.880><c> really</c><00:57:18.039><c> know</c><00:57:18.319><c> but</c>"
+    },
+    {
+      "start": 3438.589,
+      "duration": 0.0,
+      "text": "some challenges I don't really know but"
+    },
+    {
+      "start": 3438.599,
+      "duration": 0.0,
+      "text": "some challenges I don't really know but if<00:57:18.720><c> you</c><00:57:18.880><c> follow</c><00:57:19.280><c> the</c><00:57:19.440><c> simple</c><00:57:19.760><c> computation</c>"
+    },
+    {
+      "start": 3440.27,
+      "duration": 0.0,
+      "text": "if you follow the simple computation"
+    },
+    {
+      "start": 3440.28,
+      "duration": 0.0,
+      "text": "if you follow the simple computation it's<00:57:20.440><c> around</c><00:57:20.680><c> 70</c><00:57:21.480><c> days</c><00:57:22.480><c> um</c><00:57:23.240><c> cost</c><00:57:24.240><c> uh</c><00:57:24.400><c> I</c><00:57:24.480><c> mean</c>"
+    },
+    {
+      "start": 3444.71,
+      "duration": 0.0,
+      "text": "it's around 70 days um cost uh I mean"
+    },
+    {
+      "start": 3444.72,
+      "duration": 0.0,
+      "text": "it's around 70 days um cost uh I mean this<00:57:25.000><c> it's</c><00:57:25.280><c> hard</c><00:57:25.520><c> to</c><00:57:26.280><c> to</c><00:57:26.520><c> approximate</c><00:57:27.079><c> but</c><00:57:27.240><c> I'm</c>"
+    },
+    {
+      "start": 3447.309,
+      "duration": 0.0,
+      "text": "this it's hard to to approximate but I'm"
+    },
+    {
+      "start": 3447.319,
+      "duration": 0.0,
+      "text": "this it's hard to to approximate but I'm just<00:57:27.480><c> going</c><00:57:27.559><c> to</c><00:57:27.720><c> say</c><00:57:27.920><c> it's</c><00:57:28.440><c> kind</c><00:57:28.559><c> of</c><00:57:28.760><c> the</c><00:57:28.960><c> rent</c>"
+    },
+    {
+      "start": 3449.43,
+      "duration": 0.0,
+      "text": "just going to say it's kind of the rent"
+    },
+    {
+      "start": 3449.44,
+      "duration": 0.0,
+      "text": "just going to say it's kind of the rent like<00:57:29.640><c> what</c><00:57:29.760><c> if</c><00:57:29.920><c> I</c><00:57:30.000><c> were</c><00:57:30.280><c> to</c><00:57:30.480><c> rent</c><00:57:31.280><c> h100s</c><00:57:32.280><c> that</c>"
+    },
+    {
+      "start": 3452.43,
+      "duration": 0.0,
+      "text": "like what if I were to rent h100s that"
+    },
+    {
+      "start": 3452.44,
+      "duration": 0.0,
+      "text": "like what if I were to rent h100s that many<00:57:32.920><c> h100s</c><00:57:33.799><c> for</c><00:57:34.400><c> that</c><00:57:34.559><c> many</c><00:57:34.880><c> days</c><00:57:35.160><c> how</c><00:57:35.280><c> much</c>"
+    },
+    {
+      "start": 3455.43,
+      "duration": 0.0,
+      "text": "many h100s for that many days how much"
+    },
+    {
+      "start": 3455.44,
+      "duration": 0.0,
+      "text": "many h100s for that many days how much will<00:57:35.599><c> I</c><00:57:35.839><c> pay</c><00:57:36.480><c> uh</c><00:57:36.599><c> h100</c><00:57:37.280><c> a</c><00:57:37.400><c> lower</c><00:57:37.720><c> bound</c><00:57:38.039><c> on</c><00:57:38.160><c> the</c>"
+    },
+    {
+      "start": 3458.51,
+      "duration": 0.0,
+      "text": "will I pay uh h100 a lower bound on the"
+    },
+    {
+      "start": 3458.52,
+      "duration": 0.0,
+      "text": "will I pay uh h100 a lower bound on the on<00:57:38.720><c> the</c><00:57:38.880><c> renting</c><00:57:39.880><c> uh</c><00:57:40.039><c> cost</c><00:57:40.280><c> of</c><00:57:40.480><c> h100</c><00:57:41.079><c> is</c><00:57:41.240><c> around</c>"
+    },
+    {
+      "start": 3461.47,
+      "duration": 0.0,
+      "text": "on the renting uh cost of h100 is around"
+    },
+    {
+      "start": 3461.48,
+      "duration": 0.0,
+      "text": "on the renting uh cost of h100 is around 2<00:57:41.760><c> hours</c><00:57:42.440><c> uh</c><00:57:42.520><c> $2</c><00:57:43.079><c> per</c><00:57:43.240><c> hour</c><00:57:44.000><c> so</c><00:57:44.160><c> if</c><00:57:44.240><c> you</c>"
+    },
+    {
+      "start": 3464.349,
+      "duration": 0.0,
+      "text": "2 hours uh $2 per hour so if you"
+    },
+    {
+      "start": 3464.359,
+      "duration": 0.0,
+      "text": "2 hours uh $2 per hour so if you multiply<00:57:44.839><c> this</c><00:57:45.000><c> by</c><00:57:45.160><c> 26</c><00:57:46.000><c> million</c><00:57:46.960><c> uh</c><00:57:47.160><c> hours</c><00:57:48.160><c> uh</c>"
+    },
+    {
+      "start": 3468.27,
+      "duration": 0.0,
+      "text": "multiply this by 26 million uh hours uh"
+    },
+    {
+      "start": 3468.28,
+      "duration": 0.0,
+      "text": "multiply this by 26 million uh hours uh you<00:57:48.480><c> get</c><00:57:48.760><c> 52</c><00:57:49.359><c> million</c><00:57:50.280><c> uh</c><00:57:50.440><c> dollars</c><00:57:51.000><c> so</c><00:57:51.240><c> they</c>"
+    },
+    {
+      "start": 3471.43,
+      "duration": 0.0,
+      "text": "you get 52 million uh dollars so they"
+    },
+    {
+      "start": 3471.44,
+      "duration": 0.0,
+      "text": "you get 52 million uh dollars so they probably<00:57:51.760><c> pay</c><00:57:52.079><c> less</c><00:57:52.280><c> than</c><00:57:52.520><c> that</c><00:57:53.200><c> but</c><00:57:53.599><c> not</c>"
+    },
+    {
+      "start": 3473.95,
+      "duration": 0.0,
+      "text": "probably pay less than that but not"
+    },
+    {
+      "start": 3473.96,
+      "duration": 0.0,
+      "text": "probably pay less than that but not actually<00:57:54.400><c> much</c><00:57:54.760><c> less</c><00:57:55.079><c> because</c><00:57:55.480><c> all</c><00:57:55.680><c> these</c><00:57:56.319><c> um</c>"
+    },
+    {
+      "start": 3477.23,
+      "duration": 0.0,
+      "text": "actually much less because all these um"
+    },
+    {
+      "start": 3477.24,
+      "duration": 0.0,
+      "text": "actually much less because all these um all<00:57:57.440><c> these</c><00:57:57.599><c> services</c><00:57:58.039><c> that</c><00:57:58.240><c> actually</c><00:57:58.440><c> rent</c>"
+    },
+    {
+      "start": 3478.71,
+      "duration": 0.0,
+      "text": "all these services that actually rent"
+    },
+    {
+      "start": 3478.72,
+      "duration": 0.0,
+      "text": "all these services that actually rent gpus<00:57:59.160><c> they</c><00:57:59.280><c> don't</c><00:57:59.480><c> make</c><00:57:59.720><c> that</c><00:57:59.839><c> much</c><00:58:00.000><c> money</c><00:58:00.520><c> so</c>"
+    },
+    {
+      "start": 3480.75,
+      "duration": 0.0,
+      "text": "gpus they don't make that much money so"
+    },
+    {
+      "start": 3480.76,
+      "duration": 0.0,
+      "text": "gpus they don't make that much money so it's<00:58:01.119><c> it's</c><00:58:01.480><c> probably</c><00:58:01.760><c> slightly</c><00:58:02.160><c> less</c><00:58:02.319><c> but</c><00:58:02.440><c> not</c>"
+    },
+    {
+      "start": 3482.589,
+      "duration": 0.0,
+      "text": "it's it's probably slightly less but not"
+    },
+    {
+      "start": 3482.599,
+      "duration": 0.0,
+      "text": "it's it's probably slightly less but not that<00:58:02.760><c> much</c><00:58:02.960><c> less</c><00:58:03.880><c> um</c><00:58:04.280><c> now</c><00:58:04.720><c> salary</c><00:58:05.720><c> I</c><00:58:05.839><c> said</c><00:58:06.160><c> 50</c>"
+    },
+    {
+      "start": 3486.71,
+      "duration": 0.0,
+      "text": "that much less um now salary I said 50"
+    },
+    {
+      "start": 3486.72,
+      "duration": 0.0,
+      "text": "that much less um now salary I said 50 employees<00:58:07.720><c> 500k</c><00:58:08.440><c> per</c>"
+    },
+    {
+      "start": 3489.589,
+      "duration": 0.0,
+      "text": "employees 500k per"
+    },
+    {
+      "start": 3489.599,
+      "duration": 0.0,
+      "text": "employees 500k per year<00:58:10.599><c> say</c><00:58:10.839><c> yeah</c><00:58:10.920><c> it's</c><00:58:11.039><c> probably</c><00:58:11.240><c> the</c><00:58:11.359><c> right</c>"
+    },
+    {
+      "start": 3491.51,
+      "duration": 0.0,
+      "text": "year say yeah it's probably the right"
+    },
+    {
+      "start": 3491.52,
+      "duration": 0.0,
+      "text": "year say yeah it's probably the right ballpark<00:58:12.200><c> 25</c><00:58:12.680><c> million</c><00:58:13.359><c> uh</c><00:58:13.440><c> so</c><00:58:13.559><c> if</c><00:58:13.640><c> you</c><00:58:13.760><c> put</c><00:58:13.960><c> all</c>"
+    },
+    {
+      "start": 3494.19,
+      "duration": 0.0,
+      "text": "ballpark 25 million uh so if you put all"
+    },
+    {
+      "start": 3494.2,
+      "duration": 0.0,
+      "text": "ballpark 25 million uh so if you put all together<00:58:14.640><c> around</c><00:58:14.960><c> 75</c><00:58:15.760><c> million</c><00:58:16.760><c> um</c><00:58:17.200><c> dollars</c>"
+    },
+    {
+      "start": 3497.51,
+      "duration": 0.0,
+      "text": "together around 75 million um dollars"
+    },
+    {
+      "start": 3497.52,
+      "duration": 0.0,
+      "text": "together around 75 million um dollars for"
+    },
+    {
+      "start": 3498.23,
+      "duration": 0.0,
+      "text": "for"
+    },
+    {
+      "start": 3498.24,
+      "duration": 0.0,
+      "text": "for training<00:58:19.240><c> uh</c><00:58:19.520><c> this</c><00:58:19.680><c> Slammer</c><00:58:20.200><c> model</c><00:58:21.079><c> I'm</c>"
+    },
+    {
+      "start": 3501.23,
+      "duration": 0.0,
+      "text": "training uh this Slammer model I'm"
+    },
+    {
+      "start": 3501.24,
+      "duration": 0.0,
+      "text": "training uh this Slammer model I'm probably<00:58:21.480><c> off</c><00:58:21.680><c> by</c><00:58:21.880><c> like</c><00:58:22.000><c> 10</c><00:58:22.200><c> million</c><00:58:22.640><c> but</c><00:58:23.079><c> but</c>"
+    },
+    {
+      "start": 3503.27,
+      "duration": 0.0,
+      "text": "probably off by like 10 million but but"
+    },
+    {
+      "start": 3503.28,
+      "duration": 0.0,
+      "text": "probably off by like 10 million but but that's<00:58:23.520><c> kind</c><00:58:23.640><c> of</c><00:58:23.920><c> right</c><00:58:24.520><c> uh</c><00:58:24.760><c> bpk</c>"
+    },
+    {
+      "start": 3507.549,
+      "duration": 0.0,
+      "text": "that's kind of right uh bpk"
+    },
+    {
+      "start": 3507.559,
+      "duration": 0.0,
+      "text": "that's kind of right uh bpk carbon<00:58:27.920><c> emitted</c><00:58:28.920><c> um</c><00:58:29.240><c> a</c><00:58:29.319><c> lot</c><00:58:29.480><c> of</c><00:58:29.640><c> people</c><00:58:30.000><c> might</c>"
+    },
+    {
+      "start": 3510.43,
+      "duration": 0.0,
+      "text": "carbon emitted um a lot of people might"
+    },
+    {
+      "start": 3510.44,
+      "duration": 0.0,
+      "text": "carbon emitted um a lot of people might ask<00:58:30.799><c> like</c><00:58:31.280><c> also</c><00:58:31.640><c> the</c><00:58:31.760><c> cost</c><00:58:32.000><c> is</c><00:58:32.119><c> not</c><00:58:32.240><c> the</c><00:58:32.319><c> only</c>"
+    },
+    {
+      "start": 3512.51,
+      "duration": 0.0,
+      "text": "ask like also the cost is not the only"
+    },
+    {
+      "start": 3512.52,
+      "duration": 0.0,
+      "text": "ask like also the cost is not the only thing<00:58:32.640><c> that</c><00:58:32.760><c> is</c><00:58:32.920><c> important</c><00:58:33.480><c> so</c><00:58:33.680><c> I</c><00:58:33.799><c> did</c><00:58:33.960><c> the</c>"
+    },
+    {
+      "start": 3514.309,
+      "duration": 0.0,
+      "text": "thing that is important so I did the"
+    },
+    {
+      "start": 3514.319,
+      "duration": 0.0,
+      "text": "thing that is important so I did the computation<00:58:35.319><c> um</c><00:58:35.920><c> it's</c><00:58:36.200><c> around</c><00:58:37.440><c> 4</c><00:58:38.440><c> uh</c><00:58:39.039><c> 4,000</c><00:58:40.039><c> um</c>"
+    },
+    {
+      "start": 3520.39,
+      "duration": 0.0,
+      "text": "computation um it's around 4 uh 4,000 um"
+    },
+    {
+      "start": 3520.4,
+      "duration": 0.0,
+      "text": "computation um it's around 4 uh 4,000 um tons<00:58:40.920><c> of</c><00:58:41.119><c> CO2</c><00:58:42.079><c> equivalent</c><00:58:43.079><c> that</c><00:58:43.240><c> is</c><00:58:43.440><c> actually</c>"
+    },
+    {
+      "start": 3523.71,
+      "duration": 0.0,
+      "text": "tons of CO2 equivalent that is actually"
+    },
+    {
+      "start": 3523.72,
+      "duration": 0.0,
+      "text": "tons of CO2 equivalent that is actually only<00:58:44.039><c> 2,000</c><00:58:44.839><c> return</c><00:58:45.119><c> tickets</c><00:58:45.440><c> from</c><00:58:45.599><c> JFK</c><00:58:46.200><c> to</c><00:58:46.760><c> uh</c>"
+    },
+    {
+      "start": 3526.87,
+      "duration": 0.0,
+      "text": "only 2,000 return tickets from JFK to uh"
+    },
+    {
+      "start": 3526.88,
+      "duration": 0.0,
+      "text": "only 2,000 return tickets from JFK to uh London<00:58:47.760><c> so</c><00:58:48.160><c> right</c><00:58:48.359><c> now</c><00:58:49.119><c> uh</c><00:58:49.319><c> carbon</c><00:58:49.640><c> emitted</c><00:58:50.000><c> is</c>"
+    },
+    {
+      "start": 3530.23,
+      "duration": 0.0,
+      "text": "London so right now uh carbon emitted is"
+    },
+    {
+      "start": 3530.24,
+      "duration": 0.0,
+      "text": "London so right now uh carbon emitted is actually<00:58:50.799><c> not</c><00:58:51.799><c> uh</c><00:58:51.920><c> I</c><00:58:51.960><c> mean</c><00:58:52.119><c> it's</c><00:58:52.359><c> huge</c><00:58:52.680><c> but</c>"
+    },
+    {
+      "start": 3532.789,
+      "duration": 0.0,
+      "text": "actually not uh I mean it's huge but"
+    },
+    {
+      "start": 3532.799,
+      "duration": 0.0,
+      "text": "actually not uh I mean it's huge but it's<00:58:53.039><c> not</c><00:58:53.359><c> like</c><00:58:53.720><c> um</c><00:58:54.880><c> meaningful</c><00:58:55.880><c> yeah</c><00:58:56.160><c> yet</c><00:58:56.760><c> I</c>"
+    },
+    {
+      "start": 3536.91,
+      "duration": 0.0,
+      "text": "it's not like um meaningful yeah yet I"
+    },
+    {
+      "start": 3536.92,
+      "duration": 0.0,
+      "text": "it's not like um meaningful yeah yet I think<00:58:57.520><c> in</c><00:58:58.200><c> maybe</c><00:58:58.760><c> GPT</c><00:58:59.280><c> 6</c><00:58:59.720><c> gpt7</c><00:59:00.720><c> once</c><00:59:00.920><c> you</c>"
+    },
+    {
+      "start": 3541.069,
+      "duration": 0.0,
+      "text": "think in maybe GPT 6 gpt7 once you"
+    },
+    {
+      "start": 3541.079,
+      "duration": 0.0,
+      "text": "think in maybe GPT 6 gpt7 once you multiply<00:59:01.559><c> this</c><00:59:01.720><c> by</c><00:59:02.039><c> 100</c><00:59:02.799><c> that</c><00:59:02.960><c> might</c><00:59:03.160><c> become</c><00:59:03.400><c> a</c>"
+    },
+    {
+      "start": 3543.51,
+      "duration": 0.0,
+      "text": "multiply this by 100 that might become a"
+    },
+    {
+      "start": 3543.52,
+      "duration": 0.0,
+      "text": "multiply this by 100 that might become a real<00:59:03.760><c> issue</c><00:59:04.359><c> right</c><00:59:04.559><c> now</c><00:59:04.720><c> it's</c><00:59:04.920><c> still</c><00:59:05.200><c> not</c><00:59:05.720><c> uh</c><00:59:05.960><c> I</c>"
+    },
+    {
+      "start": 3546.069,
+      "duration": 0.0,
+      "text": "real issue right now it's still not uh I"
+    },
+    {
+      "start": 3546.079,
+      "duration": 0.0,
+      "text": "real issue right now it's still not uh I think<00:59:06.920><c> um</c><00:59:07.200><c> an</c><00:59:07.359><c> issue</c><00:59:07.559><c> in</c><00:59:07.640><c> the</c><00:59:07.760><c> grand</c><00:59:07.960><c> scheme</c><00:59:08.200><c> of</c>"
+    },
+    {
+      "start": 3548.63,
+      "duration": 0.0,
+      "text": "think um an issue in the grand scheme of"
+    },
+    {
+      "start": 3548.64,
+      "duration": 0.0,
+      "text": "think um an issue in the grand scheme of things<00:59:09.640><c> next</c><00:59:09.920><c> model</c><00:59:10.319><c> the</c><00:59:10.440><c> way</c><00:59:10.559><c> you</c><00:59:10.640><c> should</c><00:59:11.000><c> be</c>"
+    },
+    {
+      "start": 3551.15,
+      "duration": 0.0,
+      "text": "things next model the way you should be"
+    },
+    {
+      "start": 3551.16,
+      "duration": 0.0,
+      "text": "things next model the way you should be thinking<00:59:11.440><c> about</c><00:59:11.640><c> these</c><00:59:11.799><c> models</c><00:59:12.440><c> is</c><00:59:12.559><c> that</c>"
+    },
+    {
+      "start": 3552.87,
+      "duration": 0.0,
+      "text": "thinking about these models is that"
+    },
+    {
+      "start": 3552.88,
+      "duration": 0.0,
+      "text": "thinking about these models is that every<00:59:13.079><c> new</c><00:59:13.319><c> generation</c><00:59:14.079><c> the</c><00:59:14.200><c> number</c><00:59:14.440><c> of</c><00:59:14.559><c> flops</c>"
+    },
+    {
+      "start": 3555.029,
+      "duration": 0.0,
+      "text": "every new generation the number of flops"
+    },
+    {
+      "start": 3555.039,
+      "duration": 0.0,
+      "text": "every new generation the number of flops essentially<00:59:16.039><c> uh</c><00:59:16.160><c> multiplies</c><00:59:16.680><c> 10x</c><00:59:17.359><c> or</c><00:59:17.520><c> at</c>"
+    },
+    {
+      "start": 3557.589,
+      "duration": 0.0,
+      "text": "essentially uh multiplies 10x or at"
+    },
+    {
+      "start": 3557.599,
+      "duration": 0.0,
+      "text": "essentially uh multiplies 10x or at least<00:59:17.760><c> that's</c><00:59:17.880><c> what</c><00:59:18.000><c> they</c><00:59:18.119><c> try</c><00:59:18.880><c> uh</c><00:59:19.000><c> if</c><00:59:19.119><c> they</c>"
+    },
+    {
+      "start": 3559.27,
+      "duration": 0.0,
+      "text": "least that's what they try uh if they"
+    },
+    {
+      "start": 3559.28,
+      "duration": 0.0,
+      "text": "least that's what they try uh if they have<00:59:19.599><c> enough</c><00:59:19.839><c> energy</c><00:59:20.319><c> and</c><00:59:20.440><c> if</c><00:59:20.559><c> they</c><00:59:20.640><c> can</c><00:59:20.799><c> buy</c>"
+    },
+    {
+      "start": 3560.99,
+      "duration": 0.0,
+      "text": "have enough energy and if they can buy"
+    },
+    {
+      "start": 3561.0,
+      "duration": 0.0,
+      "text": "have enough energy and if they can buy enough"
+    },
+    {
+      "start": 3562.15,
+      "duration": 0.0,
+      "text": "enough"
+    },
+    {
+      "start": 3562.16,
+      "duration": 0.0,
+      "text": "enough gpus<00:59:23.160><c> uh</c><00:59:23.400><c> great</c><00:59:23.839><c> any</c><00:59:24.039><c> question</c><00:59:24.319><c> on</c><00:59:24.559><c> these</c><00:59:24.960><c> back</c>"
+    },
+    {
+      "start": 3565.109,
+      "duration": 0.0,
+      "text": "gpus uh great any question on these back"
+    },
+    {
+      "start": 3565.119,
+      "duration": 0.0,
+      "text": "gpus uh great any question on these back of<00:59:25.280><c> the</c><00:59:25.359><c> envelope</c><00:59:25.760><c> math</c>"
+    },
+    {
+      "start": 3569.88,
+      "duration": 0.0,
+      "text": "no"
+    },
+    {
+      "start": 3571.19,
+      "duration": 0.0,
+      "text": "no"
+    },
+    {
+      "start": 3571.2,
+      "duration": 0.0,
+      "text": "no okay<00:59:32.200><c> so</c><00:59:32.440><c> now</c><00:59:32.599><c> we</c><00:59:32.799><c> talked</c><00:59:33.319><c> about</c><00:59:33.799><c> pre-training</c>"
+    },
+    {
+      "start": 3574.75,
+      "duration": 0.0,
+      "text": "okay so now we talked about pre-training"
+    },
+    {
+      "start": 3574.76,
+      "duration": 0.0,
+      "text": "okay so now we talked about pre-training I<00:59:34.880><c> wanted</c><00:59:35.119><c> to</c><00:59:35.280><c> also</c><00:59:35.760><c> chat</c><00:59:36.079><c> about</c><00:59:36.359><c> systems</c>"
+    },
+    {
+      "start": 3576.829,
+      "duration": 0.0,
+      "text": "I wanted to also chat about systems"
+    },
+    {
+      "start": 3576.839,
+      "duration": 0.0,
+      "text": "I wanted to also chat about systems because<00:59:37.000><c> now</c><00:59:37.119><c> we</c><00:59:37.280><c> know</c><00:59:37.599><c> computer</c><00:59:38.000><c> is</c><00:59:38.160><c> really</c>"
+    },
+    {
+      "start": 3578.349,
+      "duration": 0.0,
+      "text": "because now we know computer is really"
+    },
+    {
+      "start": 3578.359,
+      "duration": 0.0,
+      "text": "because now we know computer is really important<00:59:38.920><c> so</c><00:59:39.079><c> there's</c><00:59:39.200><c> a</c><00:59:39.359><c> question</c><00:59:39.599><c> of</c><00:59:39.720><c> how</c>"
+    },
+    {
+      "start": 3579.829,
+      "duration": 0.0,
+      "text": "important so there's a question of how"
+    },
+    {
+      "start": 3579.839,
+      "duration": 0.0,
+      "text": "important so there's a question of how do<00:59:39.960><c> you</c><00:59:40.119><c> optimize</c><00:59:40.760><c> the</c><00:59:41.720><c> how</c><00:59:41.799><c> do</c><00:59:41.920><c> you</c><00:59:42.039><c> optimize</c>"
+    },
+    {
+      "start": 3582.39,
+      "duration": 0.0,
+      "text": "do you optimize the how do you optimize"
+    },
+    {
+      "start": 3582.4,
+      "duration": 0.0,
+      "text": "do you optimize the how do you optimize your<00:59:42.520><c> computer</c><00:59:43.160><c> I</c><00:59:43.240><c> will</c><00:59:43.400><c> leave</c><00:59:43.640><c> that</c><00:59:43.760><c> for</c><00:59:44.000><c> the</c>"
+    },
+    {
+      "start": 3584.069,
+      "duration": 0.0,
+      "text": "your computer I will leave that for the"
+    },
+    {
+      "start": 3584.079,
+      "duration": 0.0,
+      "text": "your computer I will leave that for the end<00:59:44.559><c> because</c><00:59:44.680><c> I'm</c><00:59:44.799><c> not</c><00:59:44.920><c> sure</c><00:59:45.119><c> how</c><00:59:45.240><c> much</c><00:59:45.400><c> time</c>"
+    },
+    {
+      "start": 3585.589,
+      "duration": 0.0,
+      "text": "end because I'm not sure how much time"
+    },
+    {
+      "start": 3585.599,
+      "duration": 0.0,
+      "text": "end because I'm not sure how much time we<00:59:45.680><c> will</c><00:59:45.920><c> have</c><00:59:46.319><c> I</c><00:59:46.400><c> think</c><00:59:46.520><c> it's</c><00:59:46.720><c> important</c><00:59:47.160><c> but</c>"
+    },
+    {
+      "start": 3587.349,
+      "duration": 0.0,
+      "text": "we will have I think it's important but"
+    },
+    {
+      "start": 3587.359,
+      "duration": 0.0,
+      "text": "we will have I think it's important but hopefully<00:59:47.920><c> I</c><00:59:48.079><c> I'll</c><00:59:48.200><c> be</c><00:59:48.319><c> able</c><00:59:48.559><c> to</c><00:59:49.039><c> to</c><00:59:49.240><c> talk</c>"
+    },
+    {
+      "start": 3589.39,
+      "duration": 0.0,
+      "text": "hopefully I I'll be able to to talk"
+    },
+    {
+      "start": 3589.4,
+      "duration": 0.0,
+      "text": "hopefully I I'll be able to to talk about<00:59:49.599><c> it</c><00:59:49.799><c> later</c><00:59:50.440><c> it's</c><00:59:50.680><c> slightly</c><00:59:51.160><c> different</c>"
+    },
+    {
+      "start": 3591.63,
+      "duration": 0.0,
+      "text": "about it later it's slightly different"
+    },
+    {
+      "start": 3591.64,
+      "duration": 0.0,
+      "text": "about it later it's slightly different than<00:59:52.400><c> what</c><00:59:52.520><c> we've</c><00:59:52.680><c> been</c><00:59:52.839><c> talking</c><00:59:53.119><c> about</c><00:59:53.400><c> right</c>"
+    },
+    {
+      "start": 3593.549,
+      "duration": 0.0,
+      "text": "than what we've been talking about right"
+    },
+    {
+      "start": 3593.559,
+      "duration": 0.0,
+      "text": "than what we've been talking about right now<00:59:54.039><c> so</c><00:59:54.160><c> I'll</c><00:59:54.319><c> move</c><00:59:54.520><c> on</c><00:59:54.599><c> to</c><00:59:54.799><c> post</c><00:59:55.039><c> training</c><00:59:55.359><c> for</c>"
+    },
+    {
+      "start": 3595.51,
+      "duration": 0.0,
+      "text": "now so I'll move on to post training for"
+    },
+    {
+      "start": 3595.52,
+      "duration": 0.0,
+      "text": "now so I'll move on to post training for now"
+    },
+    {
+      "start": 3596.63,
+      "duration": 0.0,
+      "text": "now"
+    },
+    {
+      "start": 3596.64,
+      "duration": 0.0,
+      "text": "now so<00:59:56.799><c> the</c><00:59:56.960><c> task</c><00:59:57.200><c> of</c><00:59:57.319><c> post</c><00:59:57.640><c> training</c><00:59:58.640><c> ER</c><00:59:59.200><c> the</c>"
+    },
+    {
+      "start": 3599.309,
+      "duration": 0.0,
+      "text": "so the task of post training ER the"
+    },
+    {
+      "start": 3599.319,
+      "duration": 0.0,
+      "text": "so the task of post training ER the reason<00:59:59.599><c> why</c><00:59:59.720><c> we</c><00:59:59.799><c> need</c><00:59:59.920><c> to</c><01:00:00.039><c> do</c><01:00:00.160><c> Post</c><01:00:00.400><c> training</c>"
+    },
+    {
+      "start": 3601.029,
+      "duration": 0.0,
+      "text": "reason why we need to do Post training"
+    },
+    {
+      "start": 3601.039,
+      "duration": 0.0,
+      "text": "reason why we need to do Post training is<01:00:01.240><c> as</c><01:00:01.359><c> I</c><01:00:01.480><c> told</c><01:00:01.640><c> you</c><01:00:01.839><c> before</c><01:00:03.000><c> um</c><01:00:04.000><c> it's</c><01:00:04.160><c> to</c><01:00:04.359><c> make</c>"
+    },
+    {
+      "start": 3604.95,
+      "duration": 0.0,
+      "text": "is as I told you before um it's to make"
+    },
+    {
+      "start": 3604.96,
+      "duration": 0.0,
+      "text": "is as I told you before um it's to make AI<01:00:05.319><c> assistants</c><01:00:06.079><c> so</c><01:00:06.319><c> language</c><01:00:06.720><c> modeling</c><01:00:07.559><c> is</c>"
+    },
+    {
+      "start": 3607.829,
+      "duration": 0.0,
+      "text": "AI assistants so language modeling is"
+    },
+    {
+      "start": 3607.839,
+      "duration": 0.0,
+      "text": "AI assistants so language modeling is not<01:00:08.839><c> uh</c><01:00:09.079><c> really</c><01:00:09.359><c> the</c><01:00:09.559><c> thing</c><01:00:09.799><c> that</c><01:00:09.920><c> you</c><01:00:10.039><c> want</c>"
+    },
+    {
+      "start": 3610.43,
+      "duration": 0.0,
+      "text": "not uh really the thing that you want"
+    },
+    {
+      "start": 3610.44,
+      "duration": 0.0,
+      "text": "not uh really the thing that you want when<01:00:10.559><c> you</c><01:00:10.680><c> have</c><01:00:10.799><c> an</c><01:00:10.960><c> AI</c><01:00:11.480><c> assistant</c><01:00:12.480><c> uh</c><01:00:12.599><c> for</c>"
+    },
+    {
+      "start": 3612.75,
+      "duration": 0.0,
+      "text": "when you have an AI assistant uh for"
+    },
+    {
+      "start": 3612.76,
+      "duration": 0.0,
+      "text": "when you have an AI assistant uh for example<01:00:13.160><c> if</c><01:00:13.240><c> you</c><01:00:13.520><c> ask</c><01:00:13.880><c> to</c><01:00:14.079><c> gbd3</c><01:00:14.720><c> which</c><01:00:14.839><c> is</c><01:00:14.920><c> a</c>"
+    },
+    {
+      "start": 3615.069,
+      "duration": 0.0,
+      "text": "example if you ask to gbd3 which is a"
+    },
+    {
+      "start": 3615.079,
+      "duration": 0.0,
+      "text": "example if you ask to gbd3 which is a purely<01:00:15.799><c> language</c><01:00:16.200><c> Model</c><01:00:16.799><c> A</c><01:00:16.920><c> pure</c><01:00:17.119><c> language</c>"
+    },
+    {
+      "start": 3617.43,
+      "duration": 0.0,
+      "text": "purely language Model A pure language"
+    },
+    {
+      "start": 3617.44,
+      "duration": 0.0,
+      "text": "purely language Model A pure language model<01:00:17.760><c> not</c><01:00:18.000><c> a</c><01:00:18.400><c> um</c><01:00:18.880><c> not</c><01:00:19.079><c> an</c><01:00:19.280><c> aligned</c><01:00:19.680><c> one</c><01:00:20.200><c> if</c><01:00:20.280><c> you</c>"
+    },
+    {
+      "start": 3620.43,
+      "duration": 0.0,
+      "text": "model not a um not an aligned one if you"
+    },
+    {
+      "start": 3620.44,
+      "duration": 0.0,
+      "text": "model not a um not an aligned one if you ask<01:00:20.599><c> a</c><01:00:20.760><c> question</c><01:00:21.000><c> like</c><01:00:21.200><c> explain</c><01:00:21.520><c> the</c><01:00:21.640><c> moon</c>"
+    },
+    {
+      "start": 3622.109,
+      "duration": 0.0,
+      "text": "ask a question like explain the moon"
+    },
+    {
+      "start": 3622.119,
+      "duration": 0.0,
+      "text": "ask a question like explain the moon landing<01:00:22.520><c> to</c><01:00:22.640><c> a</c>"
+    },
+    {
+      "start": 3623.63,
+      "duration": 0.0,
+      "text": "landing to a"
+    },
+    {
+      "start": 3623.64,
+      "duration": 0.0,
+      "text": "landing to a six-year-old<01:00:24.640><c> the</c><01:00:24.880><c> completion</c><01:00:25.359><c> that</c><01:00:25.480><c> you</c>"
+    },
+    {
+      "start": 3625.549,
+      "duration": 0.0,
+      "text": "six-year-old the completion that you"
+    },
+    {
+      "start": 3625.559,
+      "duration": 0.0,
+      "text": "six-year-old the completion that you would<01:00:25.760><c> get</c><01:00:26.319><c> is</c><01:00:26.520><c> something</c><01:00:26.799><c> like</c><01:00:26.960><c> explain</c><01:00:27.280><c> the</c>"
+    },
+    {
+      "start": 3627.39,
+      "duration": 0.0,
+      "text": "would get is something like explain the"
+    },
+    {
+      "start": 3627.4,
+      "duration": 0.0,
+      "text": "would get is something like explain the theory<01:00:27.680><c> of</c><01:00:27.880><c> gravity</c><01:00:28.280><c> to</c><01:00:28.400><c> a</c><01:00:28.520><c> six-year-old</c>"
+    },
+    {
+      "start": 3629.349,
+      "duration": 0.0,
+      "text": "theory of gravity to a six-year-old"
+    },
+    {
+      "start": 3629.359,
+      "duration": 0.0,
+      "text": "theory of gravity to a six-year-old because<01:00:29.559><c> what</c><01:00:29.640><c> it</c><01:00:29.799><c> learned</c><01:00:30.160><c> is</c><01:00:30.240><c> that</c><01:00:30.480><c> on</c><01:00:30.760><c> on</c><01:00:30.960><c> on</c>"
+    },
+    {
+      "start": 3631.15,
+      "duration": 0.0,
+      "text": "because what it learned is that on on on"
+    },
+    {
+      "start": 3631.16,
+      "duration": 0.0,
+      "text": "because what it learned is that on on on internet<01:00:31.559><c> if</c><01:00:31.640><c> you</c><01:00:31.799><c> have</c><01:00:32.160><c> one</c><01:00:32.480><c> question</c><01:00:33.039><c> you</c>"
+    },
+    {
+      "start": 3633.19,
+      "duration": 0.0,
+      "text": "internet if you have one question you"
+    },
+    {
+      "start": 3633.2,
+      "duration": 0.0,
+      "text": "internet if you have one question you usually<01:00:33.599><c> have</c><01:00:34.079><c> maybe</c><01:00:34.319><c> another</c><01:00:34.599><c> bullet</c><01:00:34.920><c> point</c>"
+    },
+    {
+      "start": 3635.309,
+      "duration": 0.0,
+      "text": "usually have maybe another bullet point"
+    },
+    {
+      "start": 3635.319,
+      "duration": 0.0,
+      "text": "usually have maybe another bullet point of<01:00:35.520><c> other</c><01:00:35.799><c> similar</c><01:00:36.200><c> questions</c><01:00:36.839><c> you</c><01:00:36.960><c> don't</c>"
+    },
+    {
+      "start": 3637.15,
+      "duration": 0.0,
+      "text": "of other similar questions you don't"
+    },
+    {
+      "start": 3637.16,
+      "duration": 0.0,
+      "text": "of other similar questions you don't usually<01:00:37.440><c> have</c><01:00:37.599><c> question</c><01:00:37.839><c> and</c><01:00:37.960><c> then</c><01:00:38.160><c> answer</c>"
+    },
+    {
+      "start": 3638.47,
+      "duration": 0.0,
+      "text": "usually have question and then answer"
+    },
+    {
+      "start": 3638.48,
+      "duration": 0.0,
+      "text": "usually have question and then answer later<01:00:39.319><c> uh</c><01:00:39.480><c> this</c><01:00:39.599><c> is</c><01:00:39.760><c> not</c><01:00:39.960><c> what</c><01:00:40.079><c> you</c><01:00:40.200><c> want</c><01:00:40.799><c> from</c>"
+    },
+    {
+      "start": 3641.19,
+      "duration": 0.0,
+      "text": "later uh this is not what you want from"
+    },
+    {
+      "start": 3641.2,
+      "duration": 0.0,
+      "text": "later uh this is not what you want from an<01:00:41.359><c> AI</c><01:00:41.960><c> assistant</c><01:00:42.960><c> so</c><01:00:43.240><c> how</c><01:00:43.400><c> do</c><01:00:43.599><c> we</c><01:00:44.480><c> uh</c><01:00:44.599><c> do</c><01:00:44.880><c> this</c>"
+    },
+    {
+      "start": 3645.19,
+      "duration": 0.0,
+      "text": "an AI assistant so how do we uh do this"
+    },
+    {
+      "start": 3645.2,
+      "duration": 0.0,
+      "text": "an AI assistant so how do we uh do this alignment<01:00:45.839><c> which</c><01:00:45.920><c> is</c><01:00:46.119><c> this</c><01:00:46.280><c> post</c><01:00:46.559><c> training</c>"
+    },
+    {
+      "start": 3646.91,
+      "duration": 0.0,
+      "text": "alignment which is this post training"
+    },
+    {
+      "start": 3646.92,
+      "duration": 0.0,
+      "text": "alignment which is this post training and<01:00:47.079><c> making</c><01:00:47.400><c> these</c><01:00:47.559><c> models</c>"
+    },
+    {
+      "start": 3648.51,
+      "duration": 0.0,
+      "text": "and making these models"
+    },
+    {
+      "start": 3648.52,
+      "duration": 0.0,
+      "text": "and making these models assistance<01:00:49.520><c> um</c><01:00:49.839><c> so</c><01:00:50.039><c> the</c><01:00:50.200><c> goal</c><01:00:50.760><c> of</c><01:00:51.240><c> this</c>"
+    },
+    {
+      "start": 3651.43,
+      "duration": 0.0,
+      "text": "assistance um so the goal of this"
+    },
+    {
+      "start": 3651.44,
+      "duration": 0.0,
+      "text": "assistance um so the goal of this alignment<01:00:52.000><c> is</c><01:00:52.119><c> to</c><01:00:52.319><c> basically</c><01:00:52.640><c> get</c><01:00:52.799><c> LMS</c><01:00:53.480><c> follow</c>"
+    },
+    {
+      "start": 3654.23,
+      "duration": 0.0,
+      "text": "alignment is to basically get LMS follow"
+    },
+    {
+      "start": 3654.24,
+      "duration": 0.0,
+      "text": "alignment is to basically get LMS follow the<01:00:54.400><c> instructions</c><01:00:55.280><c> that</c><01:00:55.400><c> are</c><01:00:55.559><c> given</c><01:00:56.240><c> um</c><01:00:56.599><c> by</c>"
+    },
+    {
+      "start": 3656.789,
+      "duration": 0.0,
+      "text": "the instructions that are given um by"
+    },
+    {
+      "start": 3656.799,
+      "duration": 0.0,
+      "text": "the instructions that are given um by users<01:00:57.799><c> and</c><01:00:58.200><c> and</c><01:00:58.440><c> maybe</c><01:00:59.039><c> some</c><01:00:59.400><c> designers</c><01:01:00.400><c> kind</c>"
+    },
+    {
+      "start": 3660.549,
+      "duration": 0.0,
+      "text": "users and and maybe some designers kind"
+    },
+    {
+      "start": 3660.559,
+      "duration": 0.0,
+      "text": "users and and maybe some designers kind of<01:01:00.920><c> desires</c><01:01:01.920><c> um</c><01:01:02.400><c> so</c><01:01:02.839><c> think</c><01:01:03.039><c> about</c><01:01:03.240><c> moderation</c>"
+    },
+    {
+      "start": 3663.829,
+      "duration": 0.0,
+      "text": "of desires um so think about moderation"
+    },
+    {
+      "start": 3663.839,
+      "duration": 0.0,
+      "text": "of desires um so think about moderation you<01:01:04.000><c> don't</c><01:01:04.119><c> want</c><01:01:04.240><c> the</c><01:01:04.359><c> model</c><01:01:04.839><c> like</c><01:01:05.280><c> open</c><01:01:05.559><c> ey</c>"
+    },
+    {
+      "start": 3665.71,
+      "duration": 0.0,
+      "text": "you don't want the model like open ey"
+    },
+    {
+      "start": 3665.72,
+      "duration": 0.0,
+      "text": "you don't want the model like open ey definitely<01:01:05.960><c> doesn't</c><01:01:06.160><c> want</c><01:01:06.319><c> the</c><01:01:06.440><c> model</c><01:01:06.640><c> to</c><01:01:06.760><c> say</c>"
+    },
+    {
+      "start": 3667.23,
+      "duration": 0.0,
+      "text": "definitely doesn't want the model to say"
+    },
+    {
+      "start": 3667.24,
+      "duration": 0.0,
+      "text": "definitely doesn't want the model to say stuff<01:01:07.440><c> that</c><01:01:07.559><c> is</c><01:01:07.720><c> very</c>"
+    },
+    {
+      "start": 3668.75,
+      "duration": 0.0,
+      "text": "stuff that is very"
+    },
+    {
+      "start": 3668.76,
+      "duration": 0.0,
+      "text": "stuff that is very toxic<01:01:09.760><c> um</c><01:01:10.119><c> so</c><01:01:10.400><c> here</c><01:01:10.520><c> you</c><01:01:10.599><c> see</c><01:01:10.799><c> on</c><01:01:10.880><c> the</c><01:01:11.000><c> left</c>"
+    },
+    {
+      "start": 3671.23,
+      "duration": 0.0,
+      "text": "toxic um so here you see on the left"
+    },
+    {
+      "start": 3671.24,
+      "duration": 0.0,
+      "text": "toxic um so here you see on the left hand<01:01:11.440><c> side</c><01:01:12.079><c> uh</c><01:01:12.359><c> that</c><01:01:12.480><c> when</c><01:01:12.599><c> you</c><01:01:12.760><c> ask</c><01:01:12.920><c> a</c>"
+    },
+    {
+      "start": 3673.109,
+      "duration": 0.0,
+      "text": "hand side uh that when you ask a"
+    },
+    {
+      "start": 3673.119,
+      "duration": 0.0,
+      "text": "hand side uh that when you ask a question<01:01:13.400><c> it</c><01:01:13.559><c> actually</c><01:01:13.880><c> provides</c><01:01:14.280><c> a</c><01:01:14.680><c> a</c><01:01:14.799><c> real</c>"
+    },
+    {
+      "start": 3675.069,
+      "duration": 0.0,
+      "text": "question it actually provides a a real"
+    },
+    {
+      "start": 3675.079,
+      "duration": 0.0,
+      "text": "question it actually provides a a real answer<01:01:15.400><c> so</c><01:01:15.559><c> it's</c><01:01:15.720><c> not</c><01:01:16.000><c> like</c><01:01:16.400><c> uh</c><01:01:16.559><c> before</c><01:01:16.839><c> the</c>"
+    },
+    {
+      "start": 3676.99,
+      "duration": 0.0,
+      "text": "answer so it's not like uh before the"
+    },
+    {
+      "start": 3677.0,
+      "duration": 0.0,
+      "text": "answer so it's not like uh before the llm<01:01:17.960><c> and</c><01:01:18.200><c> on</c><01:01:18.319><c> the</c><01:01:18.480><c> right</c><01:01:18.720><c> hand</c><01:01:18.960><c> side</c><01:01:19.440><c> you</c><01:01:19.559><c> see</c>"
+    },
+    {
+      "start": 3679.789,
+      "duration": 0.0,
+      "text": "llm and on the right hand side you see"
+    },
+    {
+      "start": 3679.799,
+      "duration": 0.0,
+      "text": "llm and on the right hand side you see that<01:01:20.039><c> it</c><01:01:20.200><c> would</c><01:01:20.680><c> if</c><01:01:20.799><c> you</c><01:01:21.039><c> ask</c><01:01:21.400><c> to</c><01:01:21.599><c> write</c><01:01:21.760><c> a</c>"
+    },
+    {
+      "start": 3681.87,
+      "duration": 0.0,
+      "text": "that it would if you ask to write a"
+    },
+    {
+      "start": 3681.88,
+      "duration": 0.0,
+      "text": "that it would if you ask to write a tweet<01:01:22.200><c> describing</c><01:01:22.799><c> how</c><01:01:23.440><c> a</c><01:01:23.760><c> certain</c><01:01:24.760><c> part</c><01:01:24.920><c> of</c>"
+    },
+    {
+      "start": 3685.029,
+      "duration": 0.0,
+      "text": "tweet describing how a certain part of"
+    },
+    {
+      "start": 3685.039,
+      "duration": 0.0,
+      "text": "tweet describing how a certain part of the<01:01:25.200><c> population</c><01:01:25.720><c> are</c><01:01:26.039><c> evil</c><01:01:26.599><c> it</c><01:01:26.720><c> will</c><01:01:26.920><c> say</c><01:01:27.119><c> that</c>"
+    },
+    {
+      "start": 3687.19,
+      "duration": 0.0,
+      "text": "the population are evil it will say that"
+    },
+    {
+      "start": 3687.2,
+      "duration": 0.0,
+      "text": "the population are evil it will say that it<01:01:27.319><c> cannot</c><01:01:27.640><c> do</c><01:01:27.839><c> that</c><01:01:29.079><c> um</c><01:01:30.079><c> so</c><01:01:30.400><c> that's</c><01:01:30.720><c> kind</c><01:01:30.839><c> of</c>"
+    },
+    {
+      "start": 3690.99,
+      "duration": 0.0,
+      "text": "it cannot do that um so that's kind of"
+    },
+    {
+      "start": 3691.0,
+      "duration": 0.0,
+      "text": "it cannot do that um so that's kind of this"
+    },
+    {
+      "start": 3691.71,
+      "duration": 0.0,
+      "text": "this"
+    },
+    {
+      "start": 3691.72,
+      "duration": 0.0,
+      "text": "this alignment<01:01:32.720><c> uh</c><01:01:32.839><c> the</c><01:01:32.960><c> background</c><01:01:33.480><c> here</c><01:01:34.039><c> is</c><01:01:34.559><c> that</c>"
+    },
+    {
+      "start": 3695.91,
+      "duration": 0.0,
+      "text": "alignment uh the background here is that"
+    },
+    {
+      "start": 3695.92,
+      "duration": 0.0,
+      "text": "alignment uh the background here is that uh<01:01:36.920><c> basically</c><01:01:37.400><c> the</c><01:01:37.640><c> data</c><01:01:37.960><c> that</c><01:01:38.079><c> you</c><01:01:38.200><c> want</c><01:01:38.440><c> for</c>"
+    },
+    {
+      "start": 3698.589,
+      "duration": 0.0,
+      "text": "uh basically the data that you want for"
+    },
+    {
+      "start": 3698.599,
+      "duration": 0.0,
+      "text": "uh basically the data that you want for training<01:01:38.920><c> some</c><01:01:39.079><c> of</c><01:01:39.200><c> these</c><01:01:39.440><c> models</c><01:01:40.440><c> um</c><01:01:41.240><c> is</c><01:01:41.799><c> like</c>"
+    },
+    {
+      "start": 3701.95,
+      "duration": 0.0,
+      "text": "training some of these models um is like"
+    },
+    {
+      "start": 3701.96,
+      "duration": 0.0,
+      "text": "training some of these models um is like we<01:01:42.079><c> know</c><01:01:42.280><c> what</c><01:01:42.440><c> we</c><01:01:42.559><c> want</c><01:01:42.880><c> which</c><01:01:43.000><c> is</c><01:01:43.160><c> just</c>"
+    },
+    {
+      "start": 3703.39,
+      "duration": 0.0,
+      "text": "we know what we want which is just"
+    },
+    {
+      "start": 3703.4,
+      "duration": 0.0,
+      "text": "we know what we want which is just asking<01:01:43.720><c> humans</c><01:01:44.079><c> this</c><01:01:44.160><c> is</c><01:01:44.280><c> a</c><01:01:44.440><c> question</c><01:01:44.680><c> this</c><01:01:44.799><c> is</c>"
+    },
+    {
+      "start": 3704.87,
+      "duration": 0.0,
+      "text": "asking humans this is a question this is"
+    },
+    {
+      "start": 3704.88,
+      "duration": 0.0,
+      "text": "asking humans this is a question this is the<01:01:45.039><c> answer</c><01:01:45.280><c> that</c><01:01:45.400><c> you</c><01:01:45.520><c> want</c><01:01:46.400><c> uh</c><01:01:46.559><c> but</c><01:01:46.680><c> the</c>"
+    },
+    {
+      "start": 3706.789,
+      "duration": 0.0,
+      "text": "the answer that you want uh but the"
+    },
+    {
+      "start": 3706.799,
+      "duration": 0.0,
+      "text": "the answer that you want uh but the thing<01:01:46.920><c> is</c><01:01:47.039><c> that</c><01:01:47.160><c> it's</c><01:01:47.319><c> very</c><01:01:47.480><c> expensive</c><01:01:47.880><c> to</c>"
+    },
+    {
+      "start": 3707.99,
+      "duration": 0.0,
+      "text": "thing is that it's very expensive to"
+    },
+    {
+      "start": 3708.0,
+      "duration": 0.0,
+      "text": "thing is that it's very expensive to collect<01:01:48.319><c> that</c><01:01:48.480><c> data</c><01:01:49.039><c> and</c><01:01:49.160><c> it's</c><01:01:49.359><c> hard</c><01:01:49.520><c> to</c><01:01:49.680><c> find</c>"
+    },
+    {
+      "start": 3709.95,
+      "duration": 0.0,
+      "text": "collect that data and it's hard to find"
+    },
+    {
+      "start": 3709.96,
+      "duration": 0.0,
+      "text": "collect that data and it's hard to find it<01:01:50.319><c> online</c><01:01:51.319><c> uh</c><01:01:51.480><c> in</c><01:01:51.720><c> contrast</c><01:01:52.279><c> pre-training</c>"
+    },
+    {
+      "start": 3712.87,
+      "duration": 0.0,
+      "text": "it online uh in contrast pre-training"
+    },
+    {
+      "start": 3712.88,
+      "duration": 0.0,
+      "text": "it online uh in contrast pre-training data<01:01:53.359><c> is</c><01:01:53.520><c> not</c><01:01:53.760><c> what</c><01:01:53.880><c> you</c><01:01:54.000><c> want</c><01:01:54.680><c> but</c><01:01:54.880><c> there's</c><01:01:55.039><c> a</c>"
+    },
+    {
+      "start": 3715.15,
+      "duration": 0.0,
+      "text": "data is not what you want but there's a"
+    },
+    {
+      "start": 3715.16,
+      "duration": 0.0,
+      "text": "data is not what you want but there's a lot<01:01:55.319><c> of</c><01:01:55.440><c> it</c><01:01:56.160><c> um</c><01:01:56.599><c> so</c><01:01:57.000><c> what</c><01:01:57.279><c> what</c><01:01:57.400><c> we</c><01:01:57.480><c> will</c><01:01:57.640><c> do</c><01:01:57.799><c> a</c>"
+    },
+    {
+      "start": 3717.95,
+      "duration": 0.0,
+      "text": "lot of it um so what what we will do a"
+    },
+    {
+      "start": 3717.96,
+      "duration": 0.0,
+      "text": "lot of it um so what what we will do a the<01:01:58.039><c> main</c><01:01:58.279><c> idea</c><01:01:58.839><c> is</c><01:01:59.039><c> simply</c><01:01:59.640><c> take</c><01:01:59.799><c> a</c><01:01:59.960><c> pre-train</c>"
+    },
+    {
+      "start": 3720.39,
+      "duration": 0.0,
+      "text": "the main idea is simply take a pre-train"
+    },
+    {
+      "start": 3720.4,
+      "duration": 0.0,
+      "text": "the main idea is simply take a pre-train large<01:02:00.680><c> language</c><01:02:01.000><c> model</c><01:02:01.400><c> pre-train</c><01:02:01.920><c> all</c><01:02:02.079><c> of</c>"
+    },
+    {
+      "start": 3722.19,
+      "duration": 0.0,
+      "text": "large language model pre-train all of"
+    },
+    {
+      "start": 3722.2,
+      "duration": 0.0,
+      "text": "large language model pre-train all of internet<01:02:02.680><c> and</c><01:02:02.760><c> then</c><01:02:02.880><c> you</c><01:02:03.000><c> just</c><01:02:03.119><c> fine</c><01:02:03.359><c> tune</c><01:02:03.640><c> so</c>"
+    },
+    {
+      "start": 3723.75,
+      "duration": 0.0,
+      "text": "internet and then you just fine tune so"
+    },
+    {
+      "start": 3723.76,
+      "duration": 0.0,
+      "text": "internet and then you just fine tune so you<01:02:03.880><c> just</c><01:02:04.000><c> change</c><01:02:04.240><c> a</c><01:02:04.359><c> little</c><01:02:04.520><c> bit</c><01:02:04.640><c> of</c><01:02:04.760><c> weights</c>"
+    },
+    {
+      "start": 3725.269,
+      "duration": 0.0,
+      "text": "you just change a little bit of weights"
+    },
+    {
+      "start": 3725.279,
+      "duration": 0.0,
+      "text": "you just change a little bit of weights on<01:02:05.400><c> the</c><01:02:05.559><c> type</c><01:02:05.720><c> of</c><01:02:05.880><c> data</c><01:02:06.119><c> that</c><01:02:06.279><c> you</c><01:02:06.480><c> actually</c>"
+    },
+    {
+      "start": 3726.789,
+      "duration": 0.0,
+      "text": "on the type of data that you actually"
+    },
+    {
+      "start": 3726.799,
+      "duration": 0.0,
+      "text": "on the type of data that you actually want<01:02:07.440><c> and</c><01:02:07.640><c> hopefully</c><01:02:08.119><c> given</c><01:02:08.359><c> it</c><01:02:08.520><c> you</c><01:02:08.640><c> already</c>"
+    },
+    {
+      "start": 3728.829,
+      "duration": 0.0,
+      "text": "want and hopefully given it you already"
+    },
+    {
+      "start": 3728.839,
+      "duration": 0.0,
+      "text": "want and hopefully given it you already pre-train<01:02:09.240><c> it</c><01:02:09.359><c> on</c><01:02:09.440><c> all</c><01:02:09.640><c> of</c><01:02:09.760><c> Internet</c><01:02:10.279><c> it</c>"
+    },
+    {
+      "start": 3730.43,
+      "duration": 0.0,
+      "text": "pre-train it on all of Internet it"
+    },
+    {
+      "start": 3730.44,
+      "duration": 0.0,
+      "text": "pre-train it on all of Internet it basically<01:02:10.839><c> learns</c><01:02:11.680><c> or</c><01:02:11.880><c> knows</c><01:02:12.079><c> how</c><01:02:12.200><c> to</c><01:02:12.319><c> speak</c>"
+    },
+    {
+      "start": 3732.71,
+      "duration": 0.0,
+      "text": "basically learns or knows how to speak"
+    },
+    {
+      "start": 3732.72,
+      "duration": 0.0,
+      "text": "basically learns or knows how to speak in<01:02:12.839><c> English</c><01:02:13.240><c> and</c><01:02:13.480><c> and</c><01:02:14.000><c> knows</c><01:02:14.799><c> a</c><01:02:15.039><c> standard</c><01:02:16.039><c> um</c>"
+    },
+    {
+      "start": 3736.95,
+      "duration": 0.0,
+      "text": "in English and and knows a standard um"
+    },
+    {
+      "start": 3736.96,
+      "duration": 0.0,
+      "text": "in English and and knows a standard um language<01:02:17.359><c> syntax</c><01:02:18.359><c> uh</c><01:02:18.520><c> then</c><01:02:18.640><c> you</c><01:02:18.760><c> can</c><01:02:19.079><c> really</c>"
+    },
+    {
+      "start": 3739.71,
+      "duration": 0.0,
+      "text": "language syntax uh then you can really"
+    },
+    {
+      "start": 3739.72,
+      "duration": 0.0,
+      "text": "language syntax uh then you can really find<01:02:20.000><c> tune</c><01:02:20.200><c> in</c><01:02:20.359><c> with</c><01:02:20.520><c> very</c><01:02:20.720><c> little</c>"
+    },
+    {
+      "start": 3742.43,
+      "duration": 0.0,
+      "text": "find tune in with very little"
+    },
+    {
+      "start": 3742.44,
+      "duration": 0.0,
+      "text": "find tune in with very little data<01:02:23.440><c> okay</c><01:02:23.720><c> sft</c><01:02:24.720><c> so</c><01:02:24.920><c> supervis</c><01:02:25.400><c> fine</c><01:02:25.559><c> tuning</c><01:02:26.240><c> is</c>"
+    },
+    {
+      "start": 3746.43,
+      "duration": 0.0,
+      "text": "data okay sft so supervis fine tuning is"
+    },
+    {
+      "start": 3746.44,
+      "duration": 0.0,
+      "text": "data okay sft so supervis fine tuning is really<01:02:26.680><c> exactly</c><01:02:27.039><c> what</c><01:02:27.119><c> I</c><01:02:27.279><c> just</c><01:02:27.400><c> said</c><01:02:27.680><c> which</c><01:02:27.760><c> is</c>"
+    },
+    {
+      "start": 3747.91,
+      "duration": 0.0,
+      "text": "really exactly what I just said which is"
+    },
+    {
+      "start": 3747.92,
+      "duration": 0.0,
+      "text": "really exactly what I just said which is the<01:02:28.079><c> idea</c><01:02:28.400><c> of</c><01:02:28.559><c> fine-tuning</c><01:02:29.000><c> the</c><01:02:29.119><c> large</c>"
+    },
+    {
+      "start": 3749.349,
+      "duration": 0.0,
+      "text": "the idea of fine-tuning the large"
+    },
+    {
+      "start": 3749.359,
+      "duration": 0.0,
+      "text": "the idea of fine-tuning the large language<01:02:29.680><c> model</c><01:02:30.440><c> on</c><01:02:31.319><c> uh</c><01:02:31.520><c> basically</c><01:02:31.960><c> the</c>"
+    },
+    {
+      "start": 3752.19,
+      "duration": 0.0,
+      "text": "language model on uh basically the"
+    },
+    {
+      "start": 3752.2,
+      "duration": 0.0,
+      "text": "language model on uh basically the desired<01:02:32.680><c> answers</c><01:02:33.079><c> that</c><01:02:33.200><c> are</c><01:02:33.319><c> collected</c><01:02:33.680><c> from</c>"
+    },
+    {
+      "start": 3754.19,
+      "duration": 0.0,
+      "text": "desired answers that are collected from"
+    },
+    {
+      "start": 3754.2,
+      "duration": 0.0,
+      "text": "desired answers that are collected from humans<01:02:35.200><c> um</c><01:02:35.760><c> so</c><01:02:36.039><c> why</c><01:02:36.200><c> is</c><01:02:36.279><c> it</c><01:02:36.440><c> called</c><01:02:36.640><c> supervis</c>"
+    },
+    {
+      "start": 3757.069,
+      "duration": 0.0,
+      "text": "humans um so why is it called supervis"
+    },
+    {
+      "start": 3757.079,
+      "duration": 0.0,
+      "text": "humans um so why is it called supervis fine<01:02:37.240><c> tuning</c><01:02:37.760><c> because</c><01:02:38.160><c> you</c><01:02:38.359><c> basically</c><01:02:38.640><c> want</c>"
+    },
+    {
+      "start": 3758.71,
+      "duration": 0.0,
+      "text": "fine tuning because you basically want"
+    },
+    {
+      "start": 3758.72,
+      "duration": 0.0,
+      "text": "fine tuning because you basically want to<01:02:38.839><c> do</c><01:02:39.520><c> language</c><01:02:39.920><c> modeling</c><01:02:40.599><c> on</c><01:02:40.799><c> the</c><01:02:40.960><c> real</c>"
+    },
+    {
+      "start": 3761.19,
+      "duration": 0.0,
+      "text": "to do language modeling on the real"
+    },
+    {
+      "start": 3761.2,
+      "duration": 0.0,
+      "text": "to do language modeling on the real ansers<01:02:41.559><c> so</c><01:02:41.680><c> language</c><01:02:42.039><c> modeling</c><01:02:42.359><c> is</c><01:02:42.480><c> this</c><01:02:42.640><c> like</c>"
+    },
+    {
+      "start": 3762.789,
+      "duration": 0.0,
+      "text": "ansers so language modeling is this like"
+    },
+    {
+      "start": 3762.799,
+      "duration": 0.0,
+      "text": "ansers so language modeling is this like next<01:02:43.039><c> word</c><01:02:43.279><c> prediction</c><01:02:44.279><c> and</c><01:02:44.599><c> and</c><01:02:44.720><c> that's</c><01:02:44.839><c> the</c>"
+    },
+    {
+      "start": 3764.95,
+      "duration": 0.0,
+      "text": "next word prediction and and that's the"
+    },
+    {
+      "start": 3764.96,
+      "duration": 0.0,
+      "text": "next word prediction and and that's the fine-tuning<01:02:45.480><c> part</c><01:02:45.839><c> and</c><01:02:45.960><c> then</c><01:02:46.119><c> you</c><01:02:46.200><c> want</c><01:02:46.319><c> to</c><01:02:46.480><c> do</c>"
+    },
+    {
+      "start": 3766.63,
+      "duration": 0.0,
+      "text": "fine-tuning part and then you want to do"
+    },
+    {
+      "start": 3766.64,
+      "duration": 0.0,
+      "text": "fine-tuning part and then you want to do it<01:02:46.760><c> on</c><01:02:47.240><c> desired</c><01:02:47.680><c> answers</c><01:02:48.039><c> given</c><01:02:48.279><c> by</c><01:02:48.400><c> humans</c><01:02:48.680><c> so</c>"
+    },
+    {
+      "start": 3768.829,
+      "duration": 0.0,
+      "text": "it on desired answers given by humans so"
+    },
+    {
+      "start": 3768.839,
+      "duration": 0.0,
+      "text": "it on desired answers given by humans so that's<01:02:48.960><c> why</c><01:02:49.079><c> we</c><01:02:49.160><c> call</c><01:02:49.279><c> it</c>"
+    },
+    {
+      "start": 3770.39,
+      "duration": 0.0,
+      "text": "that's why we call it"
+    },
+    {
+      "start": 3770.4,
+      "duration": 0.0,
+      "text": "that's why we call it supervis<01:02:51.400><c> so</c><01:02:51.559><c> how</c><01:02:51.680><c> do</c><01:02:51.799><c> we</c><01:02:51.880><c> collect</c><01:02:52.240><c> this</c><01:02:52.400><c> data</c>"
+    },
+    {
+      "start": 3772.87,
+      "duration": 0.0,
+      "text": "supervis so how do we collect this data"
+    },
+    {
+      "start": 3772.88,
+      "duration": 0.0,
+      "text": "supervis so how do we collect this data well<01:02:53.119><c> we</c><01:02:53.319><c> I</c><01:02:53.440><c> just</c><01:02:53.559><c> said</c><01:02:53.760><c> it</c><01:02:54.000><c> you</c><01:02:54.279><c> just</c><01:02:54.520><c> ask</c>"
+    },
+    {
+      "start": 3774.87,
+      "duration": 0.0,
+      "text": "well we I just said it you just ask"
+    },
+    {
+      "start": 3774.88,
+      "duration": 0.0,
+      "text": "well we I just said it you just ask humans<01:02:55.520><c> uh</c><01:02:55.640><c> to</c><01:02:55.839><c> to</c><01:02:55.920><c> tell</c><01:02:56.079><c> you</c><01:02:56.319><c> this</c><01:02:56.400><c> is</c><01:02:56.599><c> the</c>"
+    },
+    {
+      "start": 3776.95,
+      "duration": 0.0,
+      "text": "humans uh to to tell you this is the"
+    },
+    {
+      "start": 3776.96,
+      "duration": 0.0,
+      "text": "humans uh to to tell you this is the this<01:02:57.039><c> is</c><01:02:57.119><c> a</c><01:02:57.319><c> question</c><01:02:57.640><c> this</c><01:02:57.760><c> is</c><01:02:57.839><c> the</c><01:02:58.000><c> answer</c>"
+    },
+    {
+      "start": 3778.269,
+      "duration": 0.0,
+      "text": "this is a question this is the answer"
+    },
+    {
+      "start": 3778.279,
+      "duration": 0.0,
+      "text": "this is a question this is the answer that<01:02:58.440><c> you</c><01:02:59.119><c> uh</c><01:02:59.240><c> you</c><01:02:59.359><c> would</c><01:02:59.520><c> want</c><01:02:59.680><c> from</c><01:02:59.839><c> some</c><01:03:00.000><c> of</c>"
+    },
+    {
+      "start": 3780.15,
+      "duration": 0.0,
+      "text": "that you uh you would want from some of"
+    },
+    {
+      "start": 3780.16,
+      "duration": 0.0,
+      "text": "that you uh you would want from some of these<01:03:00.319><c> models</c><01:03:00.960><c> so</c><01:03:01.240><c> this</c><01:03:01.319><c> is</c><01:03:01.440><c> an</c><01:03:01.720><c> example</c><01:03:02.720><c> um</c>"
+    },
+    {
+      "start": 3783.029,
+      "duration": 0.0,
+      "text": "these models so this is an example um"
+    },
+    {
+      "start": 3783.039,
+      "duration": 0.0,
+      "text": "these models so this is an example um sorry<01:03:03.279><c> I</c><01:03:03.400><c> can't</c><01:03:03.599><c> read</c><01:03:03.920><c> very</c><01:03:04.039><c> well</c><01:03:04.240><c> on</c><01:03:04.359><c> my</c>"
+    },
+    {
+      "start": 3784.549,
+      "duration": 0.0,
+      "text": "sorry I can't read very well on my"
+    },
+    {
+      "start": 3784.559,
+      "duration": 0.0,
+      "text": "sorry I can't read very well on my computer<01:03:05.000><c> but</c><01:03:05.680><c> uh</c><01:03:05.839><c> my</c><01:03:06.119><c> kid</c><01:03:06.920><c> uh</c><01:03:07.039><c> needs</c><01:03:07.279><c> to</c><01:03:07.440><c> do</c><01:03:07.599><c> a</c>"
+    },
+    {
+      "start": 3787.75,
+      "duration": 0.0,
+      "text": "computer but uh my kid uh needs to do a"
+    },
+    {
+      "start": 3787.76,
+      "duration": 0.0,
+      "text": "computer but uh my kid uh needs to do a science<01:03:08.480><c> um</c><01:03:08.680><c> no</c><01:03:08.799><c> let's</c><01:03:08.960><c> read</c><01:03:09.200><c> this</c><01:03:09.319><c> one</c><01:03:09.720><c> can</c>"
+    },
+    {
+      "start": 3789.87,
+      "duration": 0.0,
+      "text": "science um no let's read this one can"
+    },
+    {
+      "start": 3789.88,
+      "duration": 0.0,
+      "text": "science um no let's read this one can you<01:03:10.079><c> write</c><01:03:10.440><c> a</c><01:03:10.640><c> short</c><01:03:11.079><c> introduction</c><01:03:11.799><c> about</c><01:03:11.960><c> the</c>"
+    },
+    {
+      "start": 3792.109,
+      "duration": 0.0,
+      "text": "you write a short introduction about the"
+    },
+    {
+      "start": 3792.119,
+      "duration": 0.0,
+      "text": "you write a short introduction about the relevance<01:03:12.480><c> of</c><01:03:12.599><c> the</c><01:03:12.720><c> term</c><01:03:12.960><c> monopsony</c><01:03:13.920><c> and</c><01:03:14.039><c> then</c>"
+    },
+    {
+      "start": 3794.15,
+      "duration": 0.0,
+      "text": "relevance of the term monopsony and then"
+    },
+    {
+      "start": 3794.16,
+      "duration": 0.0,
+      "text": "relevance of the term monopsony and then it<01:03:14.279><c> says</c><01:03:14.440><c> monopsony</c><01:03:14.960><c> refers</c><01:03:15.279><c> to</c><01:03:15.400><c> a</c><01:03:15.480><c> market</c>"
+    },
+    {
+      "start": 3795.71,
+      "duration": 0.0,
+      "text": "it says monopsony refers to a market"
+    },
+    {
+      "start": 3795.72,
+      "duration": 0.0,
+      "text": "it says monopsony refers to a market structure<01:03:16.119><c> blah</c><01:03:16.279><c> blah</c><01:03:16.480><c> blah</c><01:03:16.640><c> and</c><01:03:16.720><c> that's</c><01:03:16.799><c> a</c>"
+    },
+    {
+      "start": 3796.91,
+      "duration": 0.0,
+      "text": "structure blah blah blah and that's a"
+    },
+    {
+      "start": 3796.92,
+      "duration": 0.0,
+      "text": "structure blah blah blah and that's a human<01:03:17.160><c> that</c><01:03:17.319><c> wrote</c><01:03:17.920><c> that</c><01:03:18.920><c> um</c><01:03:19.359><c> so</c><01:03:19.559><c> actually</c>"
+    },
+    {
+      "start": 3799.71,
+      "duration": 0.0,
+      "text": "human that wrote that um so actually"
+    },
+    {
+      "start": 3799.72,
+      "duration": 0.0,
+      "text": "human that wrote that um so actually this<01:03:19.839><c> is</c><01:03:20.000><c> open</c><01:03:20.279><c> Assistant</c><01:03:20.839><c> which</c><01:03:21.000><c> was</c><01:03:21.200><c> a</c><01:03:21.680><c> a</c><01:03:21.960><c> way</c>"
+    },
+    {
+      "start": 3802.15,
+      "duration": 0.0,
+      "text": "this is open Assistant which was a a way"
+    },
+    {
+      "start": 3802.16,
+      "duration": 0.0,
+      "text": "this is open Assistant which was a a way to<01:03:22.559><c> collect</c><01:03:24.000><c> um</c><01:03:25.000><c> uh</c><01:03:25.279><c> data</c><01:03:25.799><c> online</c><01:03:26.520><c> by</c>"
+    },
+    {
+      "start": 3807.349,
+      "duration": 0.0,
+      "text": "to collect um uh data online by"
+    },
+    {
+      "start": 3807.359,
+      "duration": 0.0,
+      "text": "to collect um uh data online by humans<01:03:28.359><c> so</c><01:03:28.839><c> this</c><01:03:29.279><c> type</c><01:03:29.520><c> of</c><01:03:29.839><c> supervised</c><01:03:30.359><c> fine</c>"
+    },
+    {
+      "start": 3810.51,
+      "duration": 0.0,
+      "text": "humans so this type of supervised fine"
+    },
+    {
+      "start": 3810.52,
+      "duration": 0.0,
+      "text": "humans so this type of supervised fine tuning<01:03:30.760><c> or</c><01:03:30.920><c> alignment</c><01:03:31.520><c> is</c><01:03:31.720><c> really</c><01:03:32.000><c> the</c><01:03:32.160><c> key</c><01:03:32.520><c> of</c>"
+    },
+    {
+      "start": 3812.67,
+      "duration": 0.0,
+      "text": "tuning or alignment is really the key of"
+    },
+    {
+      "start": 3812.68,
+      "duration": 0.0,
+      "text": "tuning or alignment is really the key of Chad<01:03:33.000><c> GPT</c><01:03:34.000><c> this</c><01:03:34.160><c> is</c><01:03:34.400><c> what</c><01:03:34.599><c> made</c><01:03:35.319><c> uh</c><01:03:35.440><c> the</c><01:03:35.599><c> big</c>"
+    },
+    {
+      "start": 3815.789,
+      "duration": 0.0,
+      "text": "Chad GPT this is what made uh the big"
+    },
+    {
+      "start": 3815.799,
+      "duration": 0.0,
+      "text": "Chad GPT this is what made uh the big jump<01:03:36.160><c> from</c><01:03:36.400><c> gpt3</c><01:03:37.200><c> which</c><01:03:37.319><c> was</c><01:03:37.480><c> mostly</c>"
+    },
+    {
+      "start": 3817.829,
+      "duration": 0.0,
+      "text": "jump from gpt3 which was mostly"
+    },
+    {
+      "start": 3817.839,
+      "duration": 0.0,
+      "text": "jump from gpt3 which was mostly something<01:03:38.119><c> that</c><01:03:38.279><c> was</c><01:03:38.440><c> known</c><01:03:38.799><c> by</c><01:03:38.960><c> AI</c>"
+    },
+    {
+      "start": 3819.269,
+      "duration": 0.0,
+      "text": "something that was known by AI"
+    },
+    {
+      "start": 3819.279,
+      "duration": 0.0,
+      "text": "something that was known by AI researchers<01:03:40.240><c> to</c><01:03:40.480><c> Chad</c><01:03:40.760><c> GPT</c><01:03:41.559><c> which</c><01:03:41.799><c> became</c>"
+    },
+    {
+      "start": 3822.069,
+      "duration": 0.0,
+      "text": "researchers to Chad GPT which became"
+    },
+    {
+      "start": 3822.079,
+      "duration": 0.0,
+      "text": "researchers to Chad GPT which became known<01:03:42.440><c> by</c><01:03:42.960><c> basically</c>"
+    },
+    {
+      "start": 3824.029,
+      "duration": 0.0,
+      "text": "known by basically"
+    },
+    {
+      "start": 3824.039,
+      "duration": 0.0,
+      "text": "known by basically everyone"
+    },
+    {
+      "start": 3826.309,
+      "duration": 0.0,
+      "text": "everyone"
+    },
+    {
+      "start": 3826.319,
+      "duration": 0.0,
+      "text": "everyone um<01:03:47.319><c> so</c><01:03:48.200><c> the</c><01:03:48.400><c> problem</c><01:03:48.760><c> with</c><01:03:49.720><c> uh</c><01:03:50.720><c> human</c><01:03:51.240><c> data</c><01:03:51.680><c> is</c>"
+    },
+    {
+      "start": 3831.829,
+      "duration": 0.0,
+      "text": "um so the problem with uh human data is"
+    },
+    {
+      "start": 3831.839,
+      "duration": 0.0,
+      "text": "um so the problem with uh human data is that<01:03:52.000><c> it's</c><01:03:52.920><c> uh</c><01:03:53.079><c> very</c><01:03:53.240><c> slow</c><01:03:53.520><c> to</c><01:03:53.680><c> collect</c><01:03:54.160><c> and</c>"
+    },
+    {
+      "start": 3834.349,
+      "duration": 0.0,
+      "text": "that it's uh very slow to collect and"
+    },
+    {
+      "start": 3834.359,
+      "duration": 0.0,
+      "text": "that it's uh very slow to collect and very<01:03:54.559><c> expensive</c><01:03:56.000><c> um</c><01:03:56.440><c> so</c>"
+    },
+    {
+      "start": 3837.23,
+      "duration": 0.0,
+      "text": "very expensive um so"
+    },
+    {
+      "start": 3837.24,
+      "duration": 0.0,
+      "text": "very expensive um so one<01:03:58.240><c> possible</c><01:03:58.839><c> simple</c><01:03:59.359><c> idea</c><01:03:59.920><c> is</c><01:04:00.079><c> to</c><01:04:00.240><c> use</c><01:04:00.520><c> llms</c>"
+    },
+    {
+      "start": 3841.349,
+      "duration": 0.0,
+      "text": "one possible simple idea is to use llms"
+    },
+    {
+      "start": 3841.359,
+      "duration": 0.0,
+      "text": "one possible simple idea is to use llms to<01:04:01.599><c> scale</c><01:04:02.039><c> data</c><01:04:02.319><c> collection</c><01:04:03.279><c> uh</c><01:04:03.359><c> so</c><01:04:03.559><c> that's</c>"
+    },
+    {
+      "start": 3843.789,
+      "duration": 0.0,
+      "text": "to scale data collection uh so that's"
+    },
+    {
+      "start": 3843.799,
+      "duration": 0.0,
+      "text": "to scale data collection uh so that's exactly<01:04:04.119><c> what</c><01:04:04.240><c> we</c><01:04:04.400><c> did</c><01:04:04.559><c> with</c><01:04:04.760><c> alpaca</c><01:04:05.760><c> uh</c><01:04:05.920><c> one</c>"
+    },
+    {
+      "start": 3846.069,
+      "duration": 0.0,
+      "text": "exactly what we did with alpaca uh one"
+    },
+    {
+      "start": 3846.079,
+      "duration": 0.0,
+      "text": "exactly what we did with alpaca uh one year<01:04:06.279><c> ago</c><01:04:06.839><c> what</c><01:04:06.960><c> we</c><01:04:07.119><c> did</c><01:04:07.279><c> is</c><01:04:07.400><c> that</c><01:04:07.599><c> we</c><01:04:07.799><c> asked</c><01:04:08.440><c> uh</c>"
+    },
+    {
+      "start": 3848.549,
+      "duration": 0.0,
+      "text": "year ago what we did is that we asked uh"
+    },
+    {
+      "start": 3848.559,
+      "duration": 0.0,
+      "text": "year ago what we did is that we asked uh humans<01:04:08.920><c> or</c><01:04:09.119><c> we</c><01:04:09.240><c> use</c><01:04:09.440><c> a</c><01:04:09.599><c> data</c><01:04:09.839><c> set</c><01:04:10.000><c> of</c><01:04:10.160><c> human</c><01:04:10.920><c> uh</c>"
+    },
+    {
+      "start": 3851.069,
+      "duration": 0.0,
+      "text": "humans or we use a data set of human uh"
+    },
+    {
+      "start": 3851.079,
+      "duration": 0.0,
+      "text": "humans or we use a data set of human uh question<01:04:11.400><c> answers</c><01:04:11.960><c> so</c><01:04:12.160><c> there</c><01:04:12.240><c> were</c><01:04:12.920><c> 175</c><01:04:13.920><c> uh</c>"
+    },
+    {
+      "start": 3854.029,
+      "duration": 0.0,
+      "text": "question answers so there were 175 uh"
+    },
+    {
+      "start": 3854.039,
+      "duration": 0.0,
+      "text": "question answers so there were 175 uh question<01:04:14.359><c> answers</c><01:04:14.799><c> here</c><01:04:15.240><c> and</c><01:04:15.359><c> we</c><01:04:15.520><c> asked</c><01:04:15.760><c> the</c>"
+    },
+    {
+      "start": 3855.829,
+      "duration": 0.0,
+      "text": "question answers here and we asked the"
+    },
+    {
+      "start": 3855.839,
+      "duration": 0.0,
+      "text": "question answers here and we asked the best<01:04:16.039><c> mod</c><01:04:16.279><c> at</c><01:04:16.400><c> the</c><01:04:16.520><c> time</c><01:04:16.680><c> so</c><01:04:17.319><c> text3</c><01:04:18.319><c> to</c>"
+    },
+    {
+      "start": 3858.51,
+      "duration": 0.0,
+      "text": "best mod at the time so text3 to"
+    },
+    {
+      "start": 3858.52,
+      "duration": 0.0,
+      "text": "best mod at the time so text3 to basically<01:04:18.920><c> generate</c><01:04:19.760><c> many</c><01:04:20.039><c> more</c><01:04:20.680><c> of</c><01:04:20.920><c> these</c>"
+    },
+    {
+      "start": 3861.15,
+      "duration": 0.0,
+      "text": "basically generate many more of these"
+    },
+    {
+      "start": 3861.16,
+      "duration": 0.0,
+      "text": "basically generate many more of these question<01:04:21.400><c> and</c><01:04:21.640><c> answers</c><01:04:22.319><c> so</c><01:04:22.480><c> all</c><01:04:22.599><c> we</c><01:04:22.760><c> did</c><01:04:22.920><c> is</c>"
+    },
+    {
+      "start": 3863.109,
+      "duration": 0.0,
+      "text": "question and answers so all we did is"
+    },
+    {
+      "start": 3863.119,
+      "duration": 0.0,
+      "text": "question and answers so all we did is like<01:04:23.359><c> this</c><01:04:23.480><c> is</c><01:04:23.799><c> what</c><01:04:23.960><c> humans</c><01:04:24.240><c> would</c><01:04:24.480><c> write</c><01:04:24.880><c> now</c>"
+    },
+    {
+      "start": 3865.15,
+      "duration": 0.0,
+      "text": "like this is what humans would write now"
+    },
+    {
+      "start": 3865.16,
+      "duration": 0.0,
+      "text": "like this is what humans would write now write<01:04:25.359><c> similar</c><01:04:26.039><c> answers</c><01:04:26.359><c> and</c><01:04:26.520><c> similar</c>"
+    },
+    {
+      "start": 3866.87,
+      "duration": 0.0,
+      "text": "write similar answers and similar"
+    },
+    {
+      "start": 3866.88,
+      "duration": 0.0,
+      "text": "write similar answers and similar questions<01:04:27.640><c> and</c><01:04:27.760><c> we</c><01:04:28.000><c> collected</c><01:04:29.200><c> 52,000</c><01:04:30.200><c> LM</c>"
+    },
+    {
+      "start": 3870.67,
+      "duration": 0.0,
+      "text": "questions and we collected 52,000 LM"
+    },
+    {
+      "start": 3870.68,
+      "duration": 0.0,
+      "text": "questions and we collected 52,000 LM generated<01:04:31.440><c> question</c><01:04:31.799><c> answers</c><01:04:32.520><c> and</c><01:04:32.640><c> then</c><01:04:32.799><c> what</c>"
+    },
+    {
+      "start": 3872.91,
+      "duration": 0.0,
+      "text": "generated question answers and then what"
+    },
+    {
+      "start": 3872.92,
+      "duration": 0.0,
+      "text": "generated question answers and then what we<01:04:33.079><c> did</c><01:04:33.200><c> is</c><01:04:33.359><c> simply</c><01:04:33.760><c> we</c><01:04:33.880><c> took</c><01:04:34.039><c> Lama</c><01:04:34.400><c> 7B</c><01:04:34.799><c> which</c>"
+    },
+    {
+      "start": 3874.91,
+      "duration": 0.0,
+      "text": "we did is simply we took Lama 7B which"
+    },
+    {
+      "start": 3874.92,
+      "duration": 0.0,
+      "text": "we did is simply we took Lama 7B which was<01:04:35.039><c> the</c><01:04:35.240><c> best</c><01:04:35.440><c> pre-train</c><01:04:35.880><c> model</c><01:04:36.119><c> at</c><01:04:36.200><c> the</c><01:04:36.359><c> time</c>"
+    },
+    {
+      "start": 3876.75,
+      "duration": 0.0,
+      "text": "was the best pre-train model at the time"
+    },
+    {
+      "start": 3876.76,
+      "duration": 0.0,
+      "text": "was the best pre-train model at the time and<01:04:36.839><c> we</c><01:04:36.960><c> just</c><01:04:37.119><c> fine-</c><01:04:37.359><c> tuned</c><01:04:37.920><c> this</c><01:04:38.079><c> with</c>"
+    },
+    {
+      "start": 3878.23,
+      "duration": 0.0,
+      "text": "and we just fine- tuned this with"
+    },
+    {
+      "start": 3878.24,
+      "duration": 0.0,
+      "text": "and we just fine- tuned this with supervised<01:04:38.680><c> fine</c><01:04:38.839><c> tuning</c><01:04:39.079><c> as</c><01:04:39.200><c> I</c><01:04:39.319><c> told</c><01:04:39.520><c> you</c><01:04:39.960><c> and</c>"
+    },
+    {
+      "start": 3880.109,
+      "duration": 0.0,
+      "text": "supervised fine tuning as I told you and"
+    },
+    {
+      "start": 3880.119,
+      "duration": 0.0,
+      "text": "supervised fine tuning as I told you and that's<01:04:40.400><c> how</c><01:04:40.559><c> we</c><01:04:40.720><c> got</c><01:04:41.400><c> um</c><01:04:41.599><c> the</c><01:04:41.720><c> Alpac</c><01:04:42.039><c> s7b</c>"
+    },
+    {
+      "start": 3883.549,
+      "duration": 0.0,
+      "text": "that's how we got um the Alpac s7b"
+    },
+    {
+      "start": 3883.559,
+      "duration": 0.0,
+      "text": "that's how we got um the Alpac s7b model<01:04:44.559><c> uh</c><01:04:45.119><c> and</c><01:04:45.319><c> this</c><01:04:45.400><c> is</c><01:04:45.520><c> the</c><01:04:45.680><c> type</c><01:04:45.880><c> of</c><01:04:46.000><c> data</c>"
+    },
+    {
+      "start": 3886.23,
+      "duration": 0.0,
+      "text": "model uh and this is the type of data"
+    },
+    {
+      "start": 3886.24,
+      "duration": 0.0,
+      "text": "model uh and this is the type of data that<01:04:46.359><c> we</c><01:04:46.480><c> collected</c><01:04:47.119><c> so</c><01:04:47.319><c> things</c><01:04:47.640><c> like</c><01:04:48.039><c> what</c>"
+    },
+    {
+      "start": 3888.23,
+      "duration": 0.0,
+      "text": "that we collected so things like what"
+    },
+    {
+      "start": 3888.24,
+      "duration": 0.0,
+      "text": "that we collected so things like what does<01:04:48.559><c> algorithm</c><01:04:49.079><c> mean</c><01:04:49.359><c> an</c><01:04:49.520><c> algorithm</c><01:04:49.960><c> is</c><01:04:50.039><c> a</c>"
+    },
+    {
+      "start": 3890.15,
+      "duration": 0.0,
+      "text": "does algorithm mean an algorithm is a"
+    },
+    {
+      "start": 3890.16,
+      "duration": 0.0,
+      "text": "does algorithm mean an algorithm is a step<01:04:50.440><c> by</c><01:04:50.640><c> a</c><01:04:51.039><c> stepbystep</c><01:04:52.039><c> uh</c><01:04:52.559><c> set</c><01:04:52.799><c> of</c>"
+    },
+    {
+      "start": 3892.95,
+      "duration": 0.0,
+      "text": "step by a stepbystep uh set of"
+    },
+    {
+      "start": 3892.96,
+      "duration": 0.0,
+      "text": "step by a stepbystep uh set of instruction<01:04:53.400><c> used</c><01:04:53.680><c> to</c><01:04:53.799><c> solve</c><01:04:54.000><c> a</c><01:04:54.160><c> problem</c><01:04:54.559><c> or</c>"
+    },
+    {
+      "start": 3894.91,
+      "duration": 0.0,
+      "text": "instruction used to solve a problem or"
+    },
+    {
+      "start": 3894.92,
+      "duration": 0.0,
+      "text": "instruction used to solve a problem or achieve<01:04:55.240><c> a</c><01:04:55.359><c> goal</c><01:04:55.680><c> blah</c><01:04:55.839><c> blah</c><01:04:56.000><c> blah</c><01:04:56.160><c> blah</c><01:04:56.440><c> so</c>"
+    },
+    {
+      "start": 3896.549,
+      "duration": 0.0,
+      "text": "achieve a goal blah blah blah blah so"
+    },
+    {
+      "start": 3896.559,
+      "duration": 0.0,
+      "text": "achieve a goal blah blah blah blah so the<01:04:56.720><c> data</c><01:04:57.000><c> is</c><01:04:57.160><c> not</c><01:04:57.480><c> actually</c><01:04:57.920><c> it's</c><01:04:58.079><c> actually</c>"
+    },
+    {
+      "start": 3898.269,
+      "duration": 0.0,
+      "text": "the data is not actually it's actually"
+    },
+    {
+      "start": 3898.279,
+      "duration": 0.0,
+      "text": "the data is not actually it's actually pretty<01:04:58.559><c> good</c><01:04:58.880><c> given</c><01:04:59.160><c> it</c><01:04:59.319><c> was</c><01:04:59.480><c> LM</c><01:04:59.839><c> generated</c><01:05:00.319><c> by</c>"
+    },
+    {
+      "start": 3900.43,
+      "duration": 0.0,
+      "text": "pretty good given it was LM generated by"
+    },
+    {
+      "start": 3900.44,
+      "duration": 0.0,
+      "text": "pretty good given it was LM generated by LMS<01:05:01.000><c> from</c><01:05:01.480><c> essentially</c><01:05:01.880><c> two</c><01:05:02.079><c> generations</c><01:05:03.000><c> ago</c>"
+    },
+    {
+      "start": 3904.15,
+      "duration": 0.0,
+      "text": "LMS from essentially two generations ago"
+    },
+    {
+      "start": 3904.16,
+      "duration": 0.0,
+      "text": "LMS from essentially two generations ago um<01:05:05.160><c> so</c><01:05:05.640><c> that</c><01:05:05.839><c> really</c><01:05:06.039><c> started</c><01:05:06.520><c> at</c><01:05:06.640><c> least</c><01:05:06.799><c> for</c>"
+    },
+    {
+      "start": 3906.99,
+      "duration": 0.0,
+      "text": "um so that really started at least for"
+    },
+    {
+      "start": 3907.0,
+      "duration": 0.0,
+      "text": "um so that really started at least for us<01:05:07.240><c> kind</c><01:05:07.359><c> of</c><01:05:07.480><c> as</c><01:05:07.640><c> an</c><01:05:07.960><c> academic</c><01:05:08.440><c> replication</c><01:05:08.880><c> of</c>"
+    },
+    {
+      "start": 3908.99,
+      "duration": 0.0,
+      "text": "us kind of as an academic replication of"
+    },
+    {
+      "start": 3909.0,
+      "duration": 0.0,
+      "text": "us kind of as an academic replication of chat<01:05:09.279><c> GPT</c><01:05:10.279><c> uh</c><01:05:10.520><c> now</c><01:05:11.000><c> it</c><01:05:11.440><c> really</c><01:05:12.119><c> there's</c><01:05:12.279><c> a</c><01:05:12.400><c> big</c>"
+    },
+    {
+      "start": 3912.549,
+      "duration": 0.0,
+      "text": "chat GPT uh now it really there's a big"
+    },
+    {
+      "start": 3912.559,
+      "duration": 0.0,
+      "text": "chat GPT uh now it really there's a big field<01:05:12.880><c> of</c><01:05:13.039><c> like</c><01:05:13.200><c> synthetic</c><01:05:13.640><c> data</c><01:05:13.920><c> generation</c>"
+    },
+    {
+      "start": 3914.91,
+      "duration": 0.0,
+      "text": "field of like synthetic data generation"
+    },
+    {
+      "start": 3914.92,
+      "duration": 0.0,
+      "text": "field of like synthetic data generation of<01:05:15.200><c> how</c><01:05:15.359><c> to</c><01:05:15.720><c> use</c><01:05:16.079><c> llms</c><01:05:16.839><c> to</c><01:05:17.039><c> basically</c><01:05:17.520><c> make</c>"
+    },
+    {
+      "start": 3918.029,
+      "duration": 0.0,
+      "text": "of how to use llms to basically make"
+    },
+    {
+      "start": 3918.039,
+      "duration": 0.0,
+      "text": "of how to use llms to basically make development<01:05:18.640><c> of</c><01:05:18.839><c> llms</c><01:05:19.720><c> faster</c><01:05:20.720><c> um</c><01:05:21.440><c> and</c><01:05:21.760><c> by</c>"
+    },
+    {
+      "start": 3921.95,
+      "duration": 0.0,
+      "text": "development of llms faster um and by"
+    },
+    {
+      "start": 3921.96,
+      "duration": 0.0,
+      "text": "development of llms faster um and by basically<01:05:22.279><c> by</c><01:05:22.400><c> decreasing</c><01:05:22.799><c> the</c><01:05:23.000><c> amount</c><01:05:23.279><c> of</c><01:05:23.520><c> of</c>"
+    },
+    {
+      "start": 3923.63,
+      "duration": 0.0,
+      "text": "basically by decreasing the amount of of"
+    },
+    {
+      "start": 3923.64,
+      "duration": 0.0,
+      "text": "basically by decreasing the amount of of human<01:05:23.880><c> hours</c><01:05:24.160><c> that</c><01:05:24.279><c> you</c><01:05:24.400><c> need</c>"
+    },
+    {
+      "start": 3926.99,
+      "duration": 0.0,
+      "text": "human hours that you need"
+    },
+    {
+      "start": 3927.0,
+      "duration": 0.0,
+      "text": "human hours that you need quantity<01:05:27.440><c> of</c><01:05:27.680><c> data</c><01:05:28.680><c> so</c><01:05:28.839><c> we</c><01:05:29.000><c> talked</c><01:05:29.240><c> about</c><01:05:29.480><c> what</c>"
+    },
+    {
+      "start": 3929.589,
+      "duration": 0.0,
+      "text": "quantity of data so we talked about what"
+    },
+    {
+      "start": 3929.599,
+      "duration": 0.0,
+      "text": "quantity of data so we talked about what type<01:05:29.760><c> of</c><01:05:29.880><c> data</c><01:05:30.119><c> and</c><01:05:30.240><c> how</c><01:05:30.359><c> we</c><01:05:30.440><c> collect</c><01:05:30.760><c> it</c><01:05:31.440><c> um</c>"
+    },
+    {
+      "start": 3931.75,
+      "duration": 0.0,
+      "text": "type of data and how we collect it um"
+    },
+    {
+      "start": 3931.76,
+      "duration": 0.0,
+      "text": "type of data and how we collect it um one<01:05:31.920><c> thing</c><01:05:32.079><c> which</c><01:05:32.200><c> is</c><01:05:32.359><c> surprising</c><01:05:32.839><c> with</c><01:05:33.039><c> sft</c>"
+    },
+    {
+      "start": 3933.95,
+      "duration": 0.0,
+      "text": "one thing which is surprising with sft"
+    },
+    {
+      "start": 3933.96,
+      "duration": 0.0,
+      "text": "one thing which is surprising with sft is<01:05:34.119><c> that</c><01:05:34.240><c> you</c><01:05:34.359><c> don't</c><01:05:34.559><c> need</c><01:05:34.760><c> that</c><01:05:34.920><c> much</c><01:05:35.240><c> data</c><01:05:36.240><c> uh</c>"
+    },
+    {
+      "start": 3936.589,
+      "duration": 0.0,
+      "text": "is that you don't need that much data uh"
+    },
+    {
+      "start": 3936.599,
+      "duration": 0.0,
+      "text": "is that you don't need that much data uh so<01:05:37.000><c> what</c><01:05:37.160><c> this</c><01:05:37.319><c> paper</c><01:05:37.559><c> showed</c><01:05:37.880><c> this</c><01:05:38.000><c> is</c><01:05:38.119><c> called</c>"
+    },
+    {
+      "start": 3938.309,
+      "duration": 0.0,
+      "text": "so what this paper showed this is called"
+    },
+    {
+      "start": 3938.319,
+      "duration": 0.0,
+      "text": "so what this paper showed this is called Lima<01:05:39.079><c> is</c><01:05:39.279><c> that</c><01:05:39.599><c> if</c><01:05:39.760><c> you</c><01:05:40.039><c> have</c><01:05:40.359><c> if</c><01:05:40.480><c> you</c><01:05:40.640><c> scale</c>"
+    },
+    {
+      "start": 3941.43,
+      "duration": 0.0,
+      "text": "Lima is that if you have if you scale"
+    },
+    {
+      "start": 3941.44,
+      "duration": 0.0,
+      "text": "Lima is that if you have if you scale the<01:05:41.720><c> amount</c><01:05:41.920><c> of</c><01:05:42.079><c> data</c><01:05:42.279><c> that</c><01:05:42.440><c> use</c><01:05:42.799><c> from</c><01:05:43.160><c> uh</c>"
+    },
+    {
+      "start": 3943.269,
+      "duration": 0.0,
+      "text": "the amount of data that use from uh"
+    },
+    {
+      "start": 3943.279,
+      "duration": 0.0,
+      "text": "the amount of data that use from uh supervised<01:05:43.720><c> fine</c><01:05:43.880><c> training</c><01:05:44.440><c> from</c><01:05:44.640><c> 2,000</c><01:05:45.240><c> to</c>"
+    },
+    {
+      "start": 3945.63,
+      "duration": 0.0,
+      "text": "supervised fine training from 2,000 to"
+    },
+    {
+      "start": 3945.64,
+      "duration": 0.0,
+      "text": "supervised fine training from 2,000 to 32,000<01:05:46.640><c> it</c><01:05:46.760><c> really</c><01:05:46.960><c> doesn't</c><01:05:47.240><c> help</c><01:05:47.480><c> much</c><01:05:47.880><c> so</c>"
+    },
+    {
+      "start": 3948.069,
+      "duration": 0.0,
+      "text": "32,000 it really doesn't help much so"
+    },
+    {
+      "start": 3948.079,
+      "duration": 0.0,
+      "text": "32,000 it really doesn't help much so here<01:05:48.240><c> scaling</c><01:05:48.559><c> laws</c><01:05:48.839><c> definitely</c><01:05:49.119><c> don't</c><01:05:49.359><c> help</c>"
+    },
+    {
+      "start": 3950.269,
+      "duration": 0.0,
+      "text": "here scaling laws definitely don't help"
+    },
+    {
+      "start": 3950.279,
+      "duration": 0.0,
+      "text": "here scaling laws definitely don't help um<01:05:50.920><c> so</c><01:05:51.279><c> the</c><01:05:51.559><c> the</c><01:05:51.680><c> intuition</c><01:05:52.279><c> here</c><01:05:52.760><c> is</c><01:05:52.920><c> that</c><01:05:53.200><c> all</c>"
+    },
+    {
+      "start": 3953.39,
+      "duration": 0.0,
+      "text": "um so the the intuition here is that all"
+    },
+    {
+      "start": 3953.4,
+      "duration": 0.0,
+      "text": "um so the the intuition here is that all you<01:05:53.720><c> learn</c><01:05:54.720><c> um</c><01:05:55.440><c> is</c><01:05:55.680><c> is</c><01:05:55.920><c> you</c><01:05:56.079><c> learn</c><01:05:56.359><c> how</c><01:05:56.480><c> to</c>"
+    },
+    {
+      "start": 3956.71,
+      "duration": 0.0,
+      "text": "you learn um is is you learn how to"
+    },
+    {
+      "start": 3956.72,
+      "duration": 0.0,
+      "text": "you learn um is is you learn how to format<01:05:57.559><c> your</c><01:05:57.760><c> desired</c><01:05:58.200><c> answers</c><01:05:58.960><c> another</c><01:05:59.240><c> way</c>"
+    },
+    {
+      "start": 3959.39,
+      "duration": 0.0,
+      "text": "format your desired answers another way"
+    },
+    {
+      "start": 3959.4,
+      "duration": 0.0,
+      "text": "format your desired answers another way of<01:05:59.480><c> saying</c><01:05:59.799><c> it</c><01:06:00.279><c> is</c><01:06:00.480><c> that</c><01:06:00.599><c> your</c><01:06:00.760><c> pre-trained</c>"
+    },
+    {
+      "start": 3961.269,
+      "duration": 0.0,
+      "text": "of saying it is that your pre-trained"
+    },
+    {
+      "start": 3961.279,
+      "duration": 0.0,
+      "text": "of saying it is that your pre-trained models<01:06:02.160><c> they</c><01:06:02.520><c> essentially</c><01:06:03.000><c> model</c><01:06:03.359><c> the</c>"
+    },
+    {
+      "start": 3963.51,
+      "duration": 0.0,
+      "text": "models they essentially model the"
+    },
+    {
+      "start": 3963.52,
+      "duration": 0.0,
+      "text": "models they essentially model the distribution<01:06:04.039><c> of</c><01:06:04.319><c> every</c><01:06:04.559><c> user</c><01:06:04.880><c> on</c><01:06:05.079><c> internet</c>"
+    },
+    {
+      "start": 3965.75,
+      "duration": 0.0,
+      "text": "distribution of every user on internet"
+    },
+    {
+      "start": 3965.76,
+      "duration": 0.0,
+      "text": "distribution of every user on internet one<01:06:06.039><c> that</c><01:06:06.279><c> might</c><01:06:06.440><c> write</c><01:06:06.680><c> bullet</c><01:06:07.000><c> points</c>"
+    },
+    {
+      "start": 3967.51,
+      "duration": 0.0,
+      "text": "one that might write bullet points"
+    },
+    {
+      "start": 3967.52,
+      "duration": 0.0,
+      "text": "one that might write bullet points another<01:06:07.839><c> one</c><01:06:08.079><c> that</c><01:06:08.240><c> might</c><01:06:08.520><c> answer</c><01:06:08.880><c> qu</c><01:06:09.240><c> answer</c>"
+    },
+    {
+      "start": 3969.589,
+      "duration": 0.0,
+      "text": "another one that might answer qu answer"
+    },
+    {
+      "start": 3969.599,
+      "duration": 0.0,
+      "text": "another one that might answer qu answer question<01:06:10.000><c> with</c><01:06:10.160><c> an</c><01:06:10.319><c> answer</c><01:06:11.000><c> so</c><01:06:11.200><c> all</c><01:06:11.400><c> you</c><01:06:11.599><c> tell</c>"
+    },
+    {
+      "start": 3971.75,
+      "duration": 0.0,
+      "text": "question with an answer so all you tell"
+    },
+    {
+      "start": 3971.76,
+      "duration": 0.0,
+      "text": "question with an answer so all you tell your<01:06:11.880><c> model</c><01:06:12.240><c> is</c><01:06:12.440><c> like</c><01:06:13.079><c> wait</c><01:06:13.359><c> you</c><01:06:13.480><c> should</c>"
+    },
+    {
+      "start": 3973.789,
+      "duration": 0.0,
+      "text": "your model is like wait you should"
+    },
+    {
+      "start": 3973.799,
+      "duration": 0.0,
+      "text": "your model is like wait you should actually<01:06:14.039><c> be</c><01:06:14.319><c> optimizing</c><01:06:14.920><c> more</c><01:06:15.119><c> for</c><01:06:15.440><c> this</c>"
+    },
+    {
+      "start": 3975.589,
+      "duration": 0.0,
+      "text": "actually be optimizing more for this"
+    },
+    {
+      "start": 3975.599,
+      "duration": 0.0,
+      "text": "actually be optimizing more for this type<01:06:15.799><c> of</c><01:06:15.920><c> user</c><01:06:16.440><c> than</c><01:06:16.640><c> another</c><01:06:16.920><c> one</c><01:06:17.200><c> so</c><01:06:17.359><c> you're</c>"
+    },
+    {
+      "start": 3977.47,
+      "duration": 0.0,
+      "text": "type of user than another one so you're"
+    },
+    {
+      "start": 3977.48,
+      "duration": 0.0,
+      "text": "type of user than another one so you're not<01:06:17.680><c> actually</c><01:06:17.960><c> teaching</c><01:06:18.480><c> it</c><01:06:18.839><c> and</c><01:06:19.000><c> you're</c><01:06:19.160><c> not</c>"
+    },
+    {
+      "start": 3979.39,
+      "duration": 0.0,
+      "text": "not actually teaching it and you're not"
+    },
+    {
+      "start": 3979.4,
+      "duration": 0.0,
+      "text": "not actually teaching it and you're not teaching<01:06:19.880><c> anything</c><01:06:20.680><c> through</c><01:06:21.200><c> this</c><01:06:21.680><c> um</c><01:06:22.160><c> sft</c><01:06:23.160><c> uh</c>"
+    },
+    {
+      "start": 3983.23,
+      "duration": 0.0,
+      "text": "teaching anything through this um sft uh"
+    },
+    {
+      "start": 3983.24,
+      "duration": 0.0,
+      "text": "teaching anything through this um sft uh so<01:06:23.440><c> supervis</c><01:06:23.880><c> fine</c><01:06:24.079><c> tuning</c><01:06:24.559><c> all</c><01:06:24.680><c> you</c><01:06:24.839><c> do</c><01:06:25.039><c> is</c>"
+    },
+    {
+      "start": 3985.15,
+      "duration": 0.0,
+      "text": "so supervis fine tuning all you do is"
+    },
+    {
+      "start": 3985.16,
+      "duration": 0.0,
+      "text": "so supervis fine tuning all you do is you<01:06:25.599><c> tell</c><01:06:25.799><c> the</c><01:06:25.920><c> model</c><01:06:26.200><c> to</c><01:06:26.359><c> kind</c><01:06:26.520><c> of</c><01:06:26.839><c> optimize</c>"
+    },
+    {
+      "start": 3987.309,
+      "duration": 0.0,
+      "text": "you tell the model to kind of optimize"
+    },
+    {
+      "start": 3987.319,
+      "duration": 0.0,
+      "text": "you tell the model to kind of optimize for<01:06:27.480><c> one</c><01:06:27.640><c> type</c><01:06:27.799><c> of</c><01:06:27.920><c> user</c><01:06:28.279><c> that</c><01:06:28.359><c> it</c><01:06:28.520><c> saw</c><01:06:28.839><c> already</c>"
+    },
+    {
+      "start": 3989.309,
+      "duration": 0.0,
+      "text": "for one type of user that it saw already"
+    },
+    {
+      "start": 3989.319,
+      "duration": 0.0,
+      "text": "for one type of user that it saw already in<01:06:29.400><c> a</c><01:06:29.559><c> pre-train</c><01:06:30.000><c> data</c><01:06:30.279><c> set</c><01:06:31.119><c> so</c><01:06:31.279><c> the</c><01:06:31.400><c> knowledge</c>"
+    },
+    {
+      "start": 3991.71,
+      "duration": 0.0,
+      "text": "in a pre-train data set so the knowledge"
+    },
+    {
+      "start": 3991.72,
+      "duration": 0.0,
+      "text": "in a pre-train data set so the knowledge is<01:06:31.880><c> already</c><01:06:32.079><c> in</c><01:06:32.160><c> the</c><01:06:32.240><c> pre-train</c><01:06:32.640><c> llm</c><01:06:33.520><c> uh</c><01:06:33.720><c> and</c>"
+    },
+    {
+      "start": 3993.829,
+      "duration": 0.0,
+      "text": "is already in the pre-train llm uh and"
+    },
+    {
+      "start": 3993.839,
+      "duration": 0.0,
+      "text": "is already in the pre-train llm uh and you<01:06:34.039><c> basically</c><01:06:34.359><c> just</c><01:06:34.480><c> specialize</c><01:06:34.920><c> to</c><01:06:35.039><c> one</c>"
+    },
+    {
+      "start": 3995.19,
+      "duration": 0.0,
+      "text": "you basically just specialize to one"
+    },
+    {
+      "start": 3995.2,
+      "duration": 0.0,
+      "text": "you basically just specialize to one type<01:06:35.359><c> of</c>"
+    },
+    {
+      "start": 3996.75,
+      "duration": 0.0,
+      "text": "type of"
+    },
+    {
+      "start": 3996.76,
+      "duration": 0.0,
+      "text": "type of user<01:06:37.760><c> great</c><01:06:38.000><c> any</c><01:06:38.160><c> question</c><01:06:38.400><c> on</c>"
+    },
+    {
+      "start": 4000.19,
+      "duration": 0.0,
+      "text": "user great any question on"
+    },
+    {
+      "start": 4000.2,
+      "duration": 0.0,
+      "text": "user great any question on sft<01:06:41.200><c> yes</c><01:06:42.279><c> so</c><01:06:43.279><c> I</c><01:06:43.400><c> know</c><01:06:43.520><c> it's</c><01:06:43.640><c> a</c><01:06:43.760><c> big</c><01:06:43.920><c> issue</c><01:06:44.200><c> with</c>"
+    },
+    {
+      "start": 4004.39,
+      "duration": 0.0,
+      "text": "sft yes so I know it's a big issue with"
+    },
+    {
+      "start": 4004.4,
+      "duration": 0.0,
+      "text": "sft yes so I know it's a big issue with synthetic<01:06:44.920><c> data</c><01:06:45.319><c> where</c><01:06:46.279><c> uh</c><01:06:46.839><c> if</c><01:06:46.920><c> you</c><01:06:47.119><c> keep</c>"
+    },
+    {
+      "start": 4007.71,
+      "duration": 0.0,
+      "text": "synthetic data where uh if you keep"
+    },
+    {
+      "start": 4007.72,
+      "duration": 0.0,
+      "text": "synthetic data where uh if you keep generating<01:06:48.279><c> data</c><01:06:48.520><c> from</c><01:06:48.680><c> the</c><01:06:48.799><c> same</c>"
+    },
+    {
+      "start": 4008.99,
+      "duration": 0.0,
+      "text": "generating data from the same"
+    },
+    {
+      "start": 4009.0,
+      "duration": 0.0,
+      "text": "generating data from the same distribution<01:06:49.599><c> eventually</c><01:06:49.960><c> you're</c><01:06:50.119><c> not</c>"
+    },
+    {
+      "start": 4010.269,
+      "duration": 0.0,
+      "text": "distribution eventually you're not"
+    },
+    {
+      "start": 4010.279,
+      "duration": 0.0,
+      "text": "distribution eventually you're not learning<01:06:50.640><c> a</c><01:06:50.760><c> new</c><01:06:50.960><c> distribution</c><01:06:51.480><c> you're</c>"
+    },
+    {
+      "start": 4011.71,
+      "duration": 0.0,
+      "text": "learning a new distribution you're"
+    },
+    {
+      "start": 4011.72,
+      "duration": 0.0,
+      "text": "learning a new distribution you're essentially<01:06:52.079><c> playing</c><01:06:52.359><c> with</c><01:06:52.480><c> it</c><01:06:52.599><c> it</c><01:06:52.760><c> just</c>"
+    },
+    {
+      "start": 4012.87,
+      "duration": 0.0,
+      "text": "essentially playing with it it just"
+    },
+    {
+      "start": 4012.88,
+      "duration": 0.0,
+      "text": "essentially playing with it it just bootstrapping<01:06:53.559><c> that</c><01:06:54.039><c> yeah</c><01:06:55.039><c> surely</c>"
+    },
+    {
+      "start": 4016.069,
+      "duration": 0.0,
+      "text": "bootstrapping that yeah surely"
+    },
+    {
+      "start": 4016.079,
+      "duration": 0.0,
+      "text": "bootstrapping that yeah surely you<01:06:56.240><c> can't</c><01:06:56.480><c> scale</c><01:06:56.839><c> that</c><01:06:56.960><c> forever</c><01:06:57.559><c> right</c><01:06:57.680><c> you</c>"
+    },
+    {
+      "start": 4017.829,
+      "duration": 0.0,
+      "text": "you can't scale that forever right you"
+    },
+    {
+      "start": 4017.839,
+      "duration": 0.0,
+      "text": "you can't scale that forever right you can't<01:06:58.079><c> keep</c><01:06:58.359><c> going</c><01:06:58.559><c> on</c><01:06:58.799><c> and</c><01:06:58.960><c> generating</c><01:06:59.440><c> from</c>"
+    },
+    {
+      "start": 4019.549,
+      "duration": 0.0,
+      "text": "can't keep going on and generating from"
+    },
+    {
+      "start": 4019.559,
+      "duration": 0.0,
+      "text": "can't keep going on and generating from the<01:06:59.680><c> same</c><01:06:59.880><c> distribution</c><01:07:00.400><c> you</c><01:07:00.520><c> hope</c><01:07:00.640><c> to</c><01:07:00.760><c> learn</c>"
+    },
+    {
+      "start": 4021.029,
+      "duration": 0.0,
+      "text": "the same distribution you hope to learn"
+    },
+    {
+      "start": 4021.039,
+      "duration": 0.0,
+      "text": "the same distribution you hope to learn something<01:07:01.400><c> new</c><01:07:01.760><c> yeah</c><01:07:02.279><c> uh</c><01:07:02.440><c> so</c><01:07:02.760><c> are</c><01:07:03.119><c> there</c><01:07:03.559><c> it's</c>"
+    },
+    {
+      "start": 4023.63,
+      "duration": 0.0,
+      "text": "something new yeah uh so are there it's"
+    },
+    {
+      "start": 4023.64,
+      "duration": 0.0,
+      "text": "something new yeah uh so are there it's an<01:07:03.799><c> active</c><01:07:04.039><c> area</c><01:07:04.279><c> of</c><01:07:04.400><c> research</c><01:07:04.960><c> but</c><01:07:05.240><c> any</c>"
+    },
+    {
+      "start": 4025.51,
+      "duration": 0.0,
+      "text": "an active area of research but any"
+    },
+    {
+      "start": 4025.52,
+      "duration": 0.0,
+      "text": "an active area of research but any thoughts<01:07:05.839><c> that</c><01:07:05.960><c> you</c><01:07:06.119><c> have</c><01:07:06.319><c> around</c><01:07:06.760><c> how</c><01:07:07.319><c> people</c>"
+    },
+    {
+      "start": 4027.51,
+      "duration": 0.0,
+      "text": "thoughts that you have around how people"
+    },
+    {
+      "start": 4027.52,
+      "duration": 0.0,
+      "text": "thoughts that you have around how people are<01:07:07.680><c> maybe</c><01:07:07.960><c> thinking</c><01:07:08.359><c> around</c><01:07:08.799><c> this</c><01:07:09.119><c> and</c><01:07:10.079><c> uh</c>"
+    },
+    {
+      "start": 4030.309,
+      "duration": 0.0,
+      "text": "are maybe thinking around this and uh"
+    },
+    {
+      "start": 4030.319,
+      "duration": 0.0,
+      "text": "are maybe thinking around this and uh better<01:07:10.599><c> ways</c><01:07:10.799><c> to</c><01:07:10.920><c> bootstrap</c><01:07:11.559><c> or</c><01:07:11.720><c> to</c><01:07:11.880><c> give</c><01:07:12.039><c> up</c>"
+    },
+    {
+      "start": 4032.15,
+      "duration": 0.0,
+      "text": "better ways to bootstrap or to give up"
+    },
+    {
+      "start": 4032.16,
+      "duration": 0.0,
+      "text": "better ways to bootstrap or to give up on<01:07:12.319><c> this</c><01:07:12.480><c> idea</c><01:07:12.799><c> and</c><01:07:13.240><c> and</c><01:07:13.400><c> realize</c><01:07:13.920><c> that</c><01:07:14.319><c> the</c>"
+    },
+    {
+      "start": 4034.47,
+      "duration": 0.0,
+      "text": "on this idea and and realize that the"
+    },
+    {
+      "start": 4034.48,
+      "duration": 0.0,
+      "text": "on this idea and and realize that the chart<01:07:14.720><c> shows</c><01:07:15.079><c> you</c><01:07:15.200><c> don't</c><01:07:15.400><c> need</c><01:07:15.599><c> that</c><01:07:15.760><c> many</c><01:07:15.920><c> so</c>"
+    },
+    {
+      "start": 4036.15,
+      "duration": 0.0,
+      "text": "chart shows you don't need that many so"
+    },
+    {
+      "start": 4036.16,
+      "duration": 0.0,
+      "text": "chart shows you don't need that many so just<01:07:16.359><c> get</c><01:07:16.520><c> humans</c><01:07:16.839><c> to</c><01:07:17.000><c> generate</c><01:07:17.400><c> 2,000</c><01:07:17.920><c> really</c>"
+    },
+    {
+      "start": 4038.19,
+      "duration": 0.0,
+      "text": "just get humans to generate 2,000 really"
+    },
+    {
+      "start": 4038.2,
+      "duration": 0.0,
+      "text": "just get humans to generate 2,000 really good<01:07:18.920><c> uh</c><01:07:19.400><c> yeah</c><01:07:20.160><c> so</c><01:07:20.359><c> that's</c><01:07:20.480><c> a</c><01:07:20.599><c> very</c><01:07:20.760><c> good</c>"
+    },
+    {
+      "start": 4040.95,
+      "duration": 0.0,
+      "text": "good uh yeah so that's a very good"
+    },
+    {
+      "start": 4040.96,
+      "duration": 0.0,
+      "text": "good uh yeah so that's a very good question<01:07:21.640><c> uh</c><01:07:21.839><c> so</c><01:07:22.039><c> for</c><01:07:22.240><c> the</c><01:07:22.400><c> data</c><01:07:22.720><c> stuff</c><01:07:23.000><c> so</c><01:07:23.200><c> I'm</c>"
+    },
+    {
+      "start": 4043.269,
+      "duration": 0.0,
+      "text": "question uh so for the data stuff so I'm"
+    },
+    {
+      "start": 4043.279,
+      "duration": 0.0,
+      "text": "question uh so for the data stuff so I'm saying<01:07:23.520><c> it's</c><01:07:23.640><c> not</c><01:07:23.799><c> that</c><01:07:23.920><c> important</c><01:07:24.200><c> for</c><01:07:24.359><c> sft</c>"
+    },
+    {
+      "start": 4044.789,
+      "duration": 0.0,
+      "text": "saying it's not that important for sft"
+    },
+    {
+      "start": 4044.799,
+      "duration": 0.0,
+      "text": "saying it's not that important for sft but<01:07:24.880><c> there</c><01:07:24.960><c> will</c><01:07:25.079><c> be</c><01:07:25.200><c> another</c><01:07:25.599><c> thing</c><01:07:25.720><c> we'll</c>"
+    },
+    {
+      "start": 4045.91,
+      "duration": 0.0,
+      "text": "but there will be another thing we'll"
+    },
+    {
+      "start": 4045.92,
+      "duration": 0.0,
+      "text": "but there will be another thing we'll talk<01:07:26.079><c> about</c><01:07:26.480><c> right</c><01:07:26.720><c> after</c><01:07:27.200><c> where</c><01:07:27.520><c> actually</c>"
+    },
+    {
+      "start": 4048.15,
+      "duration": 0.0,
+      "text": "talk about right after where actually"
+    },
+    {
+      "start": 4048.16,
+      "duration": 0.0,
+      "text": "talk about right after where actually data<01:07:28.480><c> does</c>"
+    },
+    {
+      "start": 4049.23,
+      "duration": 0.0,
+      "text": "data does"
+    },
+    {
+      "start": 4049.24,
+      "duration": 0.0,
+      "text": "data does matter<01:07:30.240><c> my</c><01:07:30.920><c> intuition</c><01:07:31.559><c> based</c><01:07:31.960><c> on</c><01:07:32.160><c> not</c><01:07:32.400><c> that</c>"
+    },
+    {
+      "start": 4052.549,
+      "duration": 0.0,
+      "text": "matter my intuition based on not that"
+    },
+    {
+      "start": 4052.559,
+      "duration": 0.0,
+      "text": "matter my intuition based on not that much<01:07:32.760><c> empirical</c><01:07:33.240><c> results</c><01:07:34.240><c> is</c><01:07:34.400><c> that</c><01:07:34.520><c> you</c><01:07:34.640><c> can</c>"
+    },
+    {
+      "start": 4054.87,
+      "duration": 0.0,
+      "text": "much empirical results is that you can"
+    },
+    {
+      "start": 4054.88,
+      "duration": 0.0,
+      "text": "much empirical results is that you can still<01:07:35.319><c> get</c><01:07:36.200><c> um</c><01:07:37.119><c> even</c><01:07:37.359><c> though</c><01:07:37.480><c> you</c><01:07:37.599><c> use</c><01:07:37.760><c> your</c>"
+    },
+    {
+      "start": 4057.91,
+      "duration": 0.0,
+      "text": "still get um even though you use your"
+    },
+    {
+      "start": 4057.92,
+      "duration": 0.0,
+      "text": "still get um even though you use your LMS<01:07:38.440><c> if</c><01:07:38.520><c> you</c><01:07:38.640><c> use</c><01:07:38.880><c> purely</c><01:07:39.200><c> LM</c><01:07:39.559><c> generated</c><01:07:40.039><c> text</c>"
+    },
+    {
+      "start": 4060.75,
+      "duration": 0.0,
+      "text": "LMS if you use purely LM generated text"
+    },
+    {
+      "start": 4060.76,
+      "duration": 0.0,
+      "text": "LMS if you use purely LM generated text and<01:07:40.880><c> you</c><01:07:41.000><c> do</c><01:07:41.240><c> that</c><01:07:41.400><c> for</c><01:07:41.599><c> like</c><01:07:41.799><c> three</c><01:07:42.039><c> four</c>"
+    },
+    {
+      "start": 4062.269,
+      "duration": 0.0,
+      "text": "and you do that for like three four"
+    },
+    {
+      "start": 4062.279,
+      "duration": 0.0,
+      "text": "and you do that for like three four generations<01:07:42.720><c> of</c><01:07:42.839><c> llms</c><01:07:43.279><c> I</c><01:07:43.400><c> agree</c><01:07:43.599><c> with</c><01:07:43.720><c> you</c>"
+    },
+    {
+      "start": 4063.829,
+      "duration": 0.0,
+      "text": "generations of llms I agree with you"
+    },
+    {
+      "start": 4063.839,
+      "duration": 0.0,
+      "text": "generations of llms I agree with you that<01:07:44.000><c> probably</c><01:07:44.200><c> you</c><01:07:44.279><c> won't</c><01:07:44.559><c> improve</c><01:07:45.000><c> much</c><01:07:46.000><c> but</c>"
+    },
+    {
+      "start": 4066.15,
+      "duration": 0.0,
+      "text": "that probably you won't improve much but"
+    },
+    {
+      "start": 4066.16,
+      "duration": 0.0,
+      "text": "that probably you won't improve much but for<01:07:46.279><c> me</c><01:07:46.440><c> what</c><01:07:46.559><c> is</c><01:07:46.720><c> important</c><01:07:47.039><c> is</c><01:07:47.160><c> how</c><01:07:47.240><c> do</c><01:07:47.319><c> you</c>"
+    },
+    {
+      "start": 4067.43,
+      "duration": 0.0,
+      "text": "for me what is important is how do you"
+    },
+    {
+      "start": 4067.44,
+      "duration": 0.0,
+      "text": "for me what is important is how do you use<01:07:47.760><c> like</c><01:07:47.920><c> human</c><01:07:48.200><c> in</c><01:07:48.319><c> the</c><01:07:48.480><c> loop</c><01:07:48.839><c> with</c><01:07:49.039><c> llms</c><01:07:49.960><c> not</c>"
+    },
+    {
+      "start": 4070.23,
+      "duration": 0.0,
+      "text": "use like human in the loop with llms not"
+    },
+    {
+      "start": 4070.24,
+      "duration": 0.0,
+      "text": "use like human in the loop with llms not purely<01:07:50.599><c> LMS</c><01:07:51.200><c> not</c><01:07:51.440><c> purely</c><01:07:52.279><c> uh</c><01:07:52.880><c> humans</c><01:07:53.240><c> but</c>"
+    },
+    {
+      "start": 4073.39,
+      "duration": 0.0,
+      "text": "purely LMS not purely uh humans but"
+    },
+    {
+      "start": 4073.4,
+      "duration": 0.0,
+      "text": "purely LMS not purely uh humans but maybe<01:07:53.640><c> what</c><01:07:53.720><c> you</c><01:07:53.799><c> can</c><01:07:53.920><c> do</c><01:07:54.079><c> is</c><01:07:54.240><c> just</c><01:07:54.440><c> have</c><01:07:54.760><c> the</c>"
+    },
+    {
+      "start": 4074.87,
+      "duration": 0.0,
+      "text": "maybe what you can do is just have the"
+    },
+    {
+      "start": 4074.88,
+      "duration": 0.0,
+      "text": "maybe what you can do is just have the model<01:07:55.520><c> generate</c><01:07:55.839><c> some</c><01:07:56.000><c> new</c><01:07:56.160><c> text</c><01:07:56.680><c> and</c><01:07:56.880><c> just</c><01:07:57.520><c> uh</c>"
+    },
+    {
+      "start": 4077.71,
+      "duration": 0.0,
+      "text": "model generate some new text and just uh"
+    },
+    {
+      "start": 4077.72,
+      "duration": 0.0,
+      "text": "model generate some new text and just uh humans<01:07:58.079><c> write</c><01:07:58.240><c> a</c><01:07:58.359><c> few</c><01:07:58.599><c> Edits</c><01:07:59.240><c> edits</c><01:07:59.520><c> are</c><01:07:59.720><c> much</c>"
+    },
+    {
+      "start": 4079.95,
+      "duration": 0.0,
+      "text": "humans write a few Edits edits are much"
+    },
+    {
+      "start": 4079.96,
+      "duration": 0.0,
+      "text": "humans write a few Edits edits are much faster<01:08:00.599><c> than</c><01:08:00.799><c> writing</c><01:08:01.079><c> the</c><01:08:01.240><c> entire</c><01:08:01.599><c> text</c><01:08:02.119><c> and</c>"
+    },
+    {
+      "start": 4082.23,
+      "duration": 0.0,
+      "text": "faster than writing the entire text and"
+    },
+    {
+      "start": 4082.24,
+      "duration": 0.0,
+      "text": "faster than writing the entire text and I<01:08:02.319><c> think</c><01:08:02.480><c> that</c><01:08:02.599><c> if</c><01:08:02.680><c> you</c><01:08:02.799><c> have</c><01:08:02.960><c> that</c><01:08:03.079><c> type</c><01:08:03.240><c> of</c>"
+    },
+    {
+      "start": 4083.39,
+      "duration": 0.0,
+      "text": "I think that if you have that type of"
+    },
+    {
+      "start": 4083.4,
+      "duration": 0.0,
+      "text": "I think that if you have that type of collaboration<01:08:04.400><c> then</c><01:08:04.680><c> from</c><01:08:04.960><c> like</c><01:08:05.119><c> kind</c><01:08:05.240><c> of</c><01:08:05.359><c> an</c>"
+    },
+    {
+      "start": 4085.51,
+      "duration": 0.0,
+      "text": "collaboration then from like kind of an"
+    },
+    {
+      "start": 4085.52,
+      "duration": 0.0,
+      "text": "collaboration then from like kind of an information<01:08:05.960><c> theoretical</c><01:08:06.440><c> point</c><01:08:06.599><c> of</c><01:08:06.720><c> view</c>"
+    },
+    {
+      "start": 4087.029,
+      "duration": 0.0,
+      "text": "information theoretical point of view"
+    },
+    {
+      "start": 4087.039,
+      "duration": 0.0,
+      "text": "information theoretical point of view you<01:08:07.200><c> still</c><01:08:07.559><c> get</c><01:08:08.000><c> additional</c><01:08:08.440><c> information</c><01:08:09.119><c> but</c>"
+    },
+    {
+      "start": 4089.23,
+      "duration": 0.0,
+      "text": "you still get additional information but"
+    },
+    {
+      "start": 4089.24,
+      "duration": 0.0,
+      "text": "you still get additional information but you<01:08:09.400><c> still</c><01:08:09.680><c> much</c><01:08:09.920><c> faster</c><01:08:10.279><c> than</c><01:08:10.400><c> if</c><01:08:10.520><c> you</c><01:08:10.680><c> use</c>"
+    },
+    {
+      "start": 4090.99,
+      "duration": 0.0,
+      "text": "you still much faster than if you use"
+    },
+    {
+      "start": 4091.0,
+      "duration": 0.0,
+      "text": "you still much faster than if you use humans<01:08:11.640><c> and</c><01:08:11.760><c> I</c><01:08:11.880><c> think</c><01:08:12.039><c> that</c><01:08:12.319><c> as</c><01:08:12.400><c> a</c><01:08:12.559><c> field</c><01:08:12.920><c> we'll</c>"
+    },
+    {
+      "start": 4093.15,
+      "duration": 0.0,
+      "text": "humans and I think that as a field we'll"
+    },
+    {
+      "start": 4093.16,
+      "duration": 0.0,
+      "text": "humans and I think that as a field we'll probably<01:08:13.440><c> move</c><01:08:13.720><c> towards</c><01:08:14.119><c> these</c><01:08:14.319><c> type</c><01:08:14.480><c> of</c>"
+    },
+    {
+      "start": 4094.63,
+      "duration": 0.0,
+      "text": "probably move towards these type of"
+    },
+    {
+      "start": 4094.64,
+      "duration": 0.0,
+      "text": "probably move towards these type of things<01:08:15.319><c> uh</c><01:08:15.440><c> which</c><01:08:15.640><c> is</c><01:08:16.640><c> um</c><01:08:16.839><c> really</c><01:08:17.120><c> just</c>"
+    },
+    {
+      "start": 4097.349,
+      "duration": 0.0,
+      "text": "things uh which is um really just"
+    },
+    {
+      "start": 4097.359,
+      "duration": 0.0,
+      "text": "things uh which is um really just finding<01:08:17.839><c> the</c><01:08:18.000><c> examples</c><01:08:18.759><c> that</c><01:08:18.880><c> are</c><01:08:19.080><c> important</c>"
+    },
+    {
+      "start": 4099.63,
+      "duration": 0.0,
+      "text": "finding the examples that are important"
+    },
+    {
+      "start": 4099.64,
+      "duration": 0.0,
+      "text": "finding the examples that are important and<01:08:19.839><c> and</c><01:08:20.239><c> asking</c><01:08:20.679><c> humans</c><01:08:21.159><c> it's</c><01:08:21.279><c> kind</c><01:08:21.400><c> of</c>"
+    },
+    {
+      "start": 4101.55,
+      "duration": 0.0,
+      "text": "and and asking humans it's kind of"
+    },
+    {
+      "start": 4101.56,
+      "duration": 0.0,
+      "text": "and and asking humans it's kind of active<01:08:21.759><c> learning</c><01:08:22.120><c> just</c><01:08:22.279><c> asking</c><01:08:22.560><c> humans</c>"
+    },
+    {
+      "start": 4102.95,
+      "duration": 0.0,
+      "text": "active learning just asking humans"
+    },
+    {
+      "start": 4102.96,
+      "duration": 0.0,
+      "text": "active learning just asking humans exactly<01:08:23.400><c> when</c><01:08:24.120><c> uh</c><01:08:24.239><c> you</c><01:08:24.400><c> need</c><01:08:24.600><c> to</c><01:08:25.040><c> to</c><01:08:25.239><c> get</c>"
+    },
+    {
+      "start": 4107.309,
+      "duration": 0.0,
+      "text": "exactly when uh you need to to get"
+    },
+    {
+      "start": 4107.319,
+      "duration": 0.0,
+      "text": "exactly when uh you need to to get inputs<01:08:28.319><c> yes</c><01:08:28.759><c> do</c><01:08:28.880><c> we</c><01:08:29.080><c> train</c><01:08:29.400><c> with</c><01:08:29.640><c> like</c><01:08:29.759><c> the</c>"
+    },
+    {
+      "start": 4109.87,
+      "duration": 0.0,
+      "text": "inputs yes do we train with like the"
+    },
+    {
+      "start": 4109.88,
+      "duration": 0.0,
+      "text": "inputs yes do we train with like the same<01:08:30.120><c> loss</c><01:08:30.400><c> function</c><01:08:30.880><c> the</c><01:08:31.000><c> same</c><01:08:31.400><c> like</c><01:08:31.640><c> General</c>"
+    },
+    {
+      "start": 4111.99,
+      "duration": 0.0,
+      "text": "same loss function the same like General"
+    },
+    {
+      "start": 4112.0,
+      "duration": 0.0,
+      "text": "same loss function the same like General training<01:08:32.359><c> algorithm</c><01:08:32.799><c> for</c><01:08:32.920><c> the</c><01:08:33.080><c> supervis</c>"
+    },
+    {
+      "start": 4113.749,
+      "duration": 0.0,
+      "text": "training algorithm for the supervis"
+    },
+    {
+      "start": 4113.759,
+      "duration": 0.0,
+      "text": "training algorithm for the supervis tuning<01:08:34.120><c> bit</c><01:08:34.359><c> as</c><01:08:34.480><c> we</c><01:08:34.600><c> do</c><01:08:34.759><c> for</c><01:08:35.040><c> the</c><01:08:35.239><c> for</c><01:08:35.400><c> the</c>"
+    },
+    {
+      "start": 4115.59,
+      "duration": 0.0,
+      "text": "tuning bit as we do for the for the"
+    },
+    {
+      "start": 4115.6,
+      "duration": 0.0,
+      "text": "tuning bit as we do for the for the pre-training<01:08:36.199><c> right</c><01:08:36.400><c> because</c><01:08:36.719><c> like</c><01:08:37.520><c> the</c>"
+    },
+    {
+      "start": 4117.669,
+      "duration": 0.0,
+      "text": "pre-training right because like the"
+    },
+    {
+      "start": 4117.679,
+      "duration": 0.0,
+      "text": "pre-training right because like the examples<01:08:38.080><c> you</c><01:08:38.239><c> showed</c><01:08:39.080><c> I</c><01:08:39.159><c> think</c><01:08:39.480><c> the</c><01:08:39.679><c> the</c>"
+    },
+    {
+      "start": 4119.829,
+      "duration": 0.0,
+      "text": "examples you showed I think the the"
+    },
+    {
+      "start": 4119.839,
+      "duration": 0.0,
+      "text": "examples you showed I think the the important<01:08:40.319><c> thing</c><01:08:40.679><c> of</c><01:08:41.480><c> the</c><01:08:42.480><c> good</c><01:08:42.640><c> examples</c><01:08:43.120><c> is</c>"
+    },
+    {
+      "start": 4123.51,
+      "duration": 0.0,
+      "text": "important thing of the good examples is"
+    },
+    {
+      "start": 4123.52,
+      "duration": 0.0,
+      "text": "important thing of the good examples is they're<01:08:43.719><c> like</c><01:08:43.880><c> supera</c><01:08:44.679><c> accurate</c><01:08:45.520><c> there's</c>"
+    },
+    {
+      "start": 4125.749,
+      "duration": 0.0,
+      "text": "they're like supera accurate there's"
+    },
+    {
+      "start": 4125.759,
+      "duration": 0.0,
+      "text": "they're like supera accurate there's these<01:08:46.000><c> more</c><01:08:46.679><c> complex</c><01:08:47.679><c> still</c><01:08:47.960><c> just</c><01:08:48.159><c> like</c><01:08:48.319><c> chain</c>"
+    },
+    {
+      "start": 4128.87,
+      "duration": 0.0,
+      "text": "these more complex still just like chain"
+    },
+    {
+      "start": 4128.88,
+      "duration": 0.0,
+      "text": "these more complex still just like chain same<01:08:49.400><c> so</c><01:08:49.600><c> that's</c><01:08:49.799><c> why</c><01:08:50.040><c> here</c><01:08:50.400><c> I</c><01:08:50.600><c> yeah</c><01:08:50.759><c> I</c><01:08:50.880><c> didn't</c>"
+    },
+    {
+      "start": 4131.03,
+      "duration": 0.0,
+      "text": "same so that's why here I yeah I didn't"
+    },
+    {
+      "start": 4131.04,
+      "duration": 0.0,
+      "text": "same so that's why here I yeah I didn't maybe<01:08:51.279><c> didn't</c><01:08:51.520><c> emphasize</c><01:08:52.040><c> enough</c><01:08:52.600><c> this</c><01:08:52.679><c> is</c>"
+    },
+    {
+      "start": 4132.829,
+      "duration": 0.0,
+      "text": "maybe didn't emphasize enough this is"
+    },
+    {
+      "start": 4132.839,
+      "duration": 0.0,
+      "text": "maybe didn't emphasize enough this is just<01:08:53.040><c> language</c><01:08:53.359><c> modeling</c><01:08:53.759><c> fine</c><01:08:54.000><c> tun</c><01:08:54.199><c> the</c><01:08:54.319><c> LM</c>"
+    },
+    {
+      "start": 4134.59,
+      "duration": 0.0,
+      "text": "just language modeling fine tun the LM"
+    },
+    {
+      "start": 4134.6,
+      "duration": 0.0,
+      "text": "just language modeling fine tun the LM with<01:08:54.719><c> language</c><01:08:55.000><c> model</c><01:08:55.279><c> on</c><01:08:55.560><c> the</c><01:08:55.719><c> desired</c>"
+    },
+    {
+      "start": 4136.149,
+      "duration": 0.0,
+      "text": "with language model on the desired"
+    },
+    {
+      "start": 4136.159,
+      "duration": 0.0,
+      "text": "with language model on the desired answers<01:08:56.679><c> so</c><01:08:56.839><c> this</c><01:08:56.960><c> is</c><01:08:57.159><c> literally</c><01:08:57.520><c> the</c><01:08:57.640><c> same</c>"
+    },
+    {
+      "start": 4137.829,
+      "duration": 0.0,
+      "text": "answers so this is literally the same"
+    },
+    {
+      "start": 4137.839,
+      "duration": 0.0,
+      "text": "answers so this is literally the same loss<01:08:58.679><c> um</c><01:08:59.279><c> it</c><01:08:59.440><c> will</c><01:08:59.600><c> be</c><01:08:59.839><c> different</c><01:09:00.719><c> in</c><01:09:00.880><c> two</c>"
+    },
+    {
+      "start": 4141.11,
+      "duration": 0.0,
+      "text": "loss um it will be different in two"
+    },
+    {
+      "start": 4141.12,
+      "duration": 0.0,
+      "text": "loss um it will be different in two seconds<01:09:01.880><c> but</c><01:09:02.120><c> the</c><01:09:02.359><c> first</c><01:09:02.640><c> step</c><01:09:02.839><c> of</c><01:09:03.000><c> sft</c><01:09:03.600><c> is</c>"
+    },
+    {
+      "start": 4143.789,
+      "duration": 0.0,
+      "text": "seconds but the first step of sft is"
+    },
+    {
+      "start": 4143.799,
+      "duration": 0.0,
+      "text": "seconds but the first step of sft is literally<01:09:04.159><c> the</c><01:09:04.279><c> same</c><01:09:04.520><c> loss</c><01:09:05.040><c> where</c><01:09:05.159><c> you</c><01:09:05.359><c> just</c>"
+    },
+    {
+      "start": 4145.55,
+      "duration": 0.0,
+      "text": "literally the same loss where you just"
+    },
+    {
+      "start": 4145.56,
+      "duration": 0.0,
+      "text": "literally the same loss where you just say<01:09:05.839><c> Okay</c><01:09:06.000><c> I</c><01:09:06.080><c> want</c><01:09:06.199><c> to</c><01:09:06.440><c> actually</c><01:09:06.719><c> specialize</c>"
+    },
+    {
+      "start": 4147.189,
+      "duration": 0.0,
+      "text": "say Okay I want to actually specialize"
+    },
+    {
+      "start": 4147.199,
+      "duration": 0.0,
+      "text": "say Okay I want to actually specialize on<01:09:07.359><c> that</c><01:09:07.520><c> type</c><01:09:07.679><c> of</c><01:09:07.839><c> data</c><01:09:08.319><c> so</c><01:09:08.520><c> there's</c><01:09:08.719><c> even</c><01:09:08.920><c> a</c>"
+    },
+    {
+      "start": 4149.07,
+      "duration": 0.0,
+      "text": "on that type of data so there's even a"
+    },
+    {
+      "start": 4149.08,
+      "duration": 0.0,
+      "text": "on that type of data so there's even a question<01:09:09.319><c> of</c><01:09:09.520><c> like</c><01:09:09.839><c> what</c><01:09:10.000><c> is</c><01:09:10.159><c> pre-training</c>"
+    },
+    {
+      "start": 4150.669,
+      "duration": 0.0,
+      "text": "question of like what is pre-training"
+    },
+    {
+      "start": 4150.679,
+      "duration": 0.0,
+      "text": "question of like what is pre-training what<01:09:10.799><c> is</c><01:09:10.920><c> post-training</c><01:09:11.480><c> because</c><01:09:11.640><c> in</c><01:09:11.759><c> reality</c>"
+    },
+    {
+      "start": 4152.03,
+      "duration": 0.0,
+      "text": "what is post-training because in reality"
+    },
+    {
+      "start": 4152.04,
+      "duration": 0.0,
+      "text": "what is post-training because in reality it's<01:09:12.120><c> just</c><01:09:12.239><c> like</c><01:09:12.359><c> a</c><01:09:12.480><c> different</c><01:09:12.759><c> data</c><01:09:13.040><c> that</c><01:09:13.159><c> you</c>"
+    },
+    {
+      "start": 4153.269,
+      "duration": 0.0,
+      "text": "it's just like a different data that you"
+    },
+    {
+      "start": 4153.279,
+      "duration": 0.0,
+      "text": "it's just like a different data that you use<01:09:13.759><c> the</c><01:09:13.880><c> reason</c><01:09:14.159><c> why</c><01:09:14.279><c> we</c><01:09:14.480><c> usually</c><01:09:14.719><c> call</c><01:09:14.880><c> it</c>"
+    },
+    {
+      "start": 4154.99,
+      "duration": 0.0,
+      "text": "use the reason why we usually call it"
+    },
+    {
+      "start": 4155.0,
+      "duration": 0.0,
+      "text": "use the reason why we usually call it post<01:09:15.239><c> training</c><01:09:15.560><c> is</c><01:09:15.640><c> that</c><01:09:15.799><c> the</c><01:09:15.880><c> way</c><01:09:16.000><c> we</c><01:09:16.120><c> collect</c>"
+    },
+    {
+      "start": 4156.39,
+      "duration": 0.0,
+      "text": "post training is that the way we collect"
+    },
+    {
+      "start": 4156.4,
+      "duration": 0.0,
+      "text": "post training is that the way we collect that<01:09:16.560><c> data</c><01:09:16.759><c> is</c><01:09:16.920><c> very</c>"
+    },
+    {
+      "start": 4158.03,
+      "duration": 0.0,
+      "text": "that data is very"
+    },
+    {
+      "start": 4158.04,
+      "duration": 0.0,
+      "text": "that data is very different<01:09:19.040><c> great</c><01:09:19.520><c> great</c><01:09:19.960><c> questions</c><01:09:20.960><c> uh</c><01:09:21.159><c> yes</c>"
+    },
+    {
+      "start": 4162.03,
+      "duration": 0.0,
+      "text": "different great great questions uh yes"
+    },
+    {
+      "start": 4162.04,
+      "duration": 0.0,
+      "text": "different great great questions uh yes maybe<01:09:22.279><c> it's</c><01:09:22.400><c> the</c><01:09:22.600><c> same</c><01:09:22.960><c> question</c><01:09:23.319><c> but</c><01:09:23.520><c> why</c>"
+    },
+    {
+      "start": 4163.669,
+      "duration": 0.0,
+      "text": "maybe it's the same question but why"
+    },
+    {
+      "start": 4163.679,
+      "duration": 0.0,
+      "text": "maybe it's the same question but why would<01:09:24.000><c> these</c><01:09:24.239><c> 2,000</c><01:09:24.920><c> examples</c><01:09:25.640><c> have</c><01:09:25.880><c> such</c><01:09:26.040><c> an</c>"
+    },
+    {
+      "start": 4166.55,
+      "duration": 0.0,
+      "text": "would these 2,000 examples have such an"
+    },
+    {
+      "start": 4166.56,
+      "duration": 0.0,
+      "text": "would these 2,000 examples have such an overweighted"
+    },
+    {
+      "start": 4168.03,
+      "duration": 0.0,
+      "text": "overweighted"
+    },
+    {
+      "start": 4168.04,
+      "duration": 0.0,
+      "text": "overweighted influence<01:09:29.040><c> you</c><01:09:29.679><c> tun</c><01:09:30.239><c> so</c><01:09:30.400><c> that's</c><01:09:30.600><c> why</c><01:09:30.839><c> we</c><01:09:31.400><c> uh</c>"
+    },
+    {
+      "start": 4171.749,
+      "duration": 0.0,
+      "text": "influence you tun so that's why we uh"
+    },
+    {
+      "start": 4171.759,
+      "duration": 0.0,
+      "text": "influence you tun so that's why we uh also<01:09:32.000><c> that's</c><01:09:32.159><c> another</c><01:09:32.400><c> reason</c><01:09:32.679><c> why</c><01:09:32.799><c> we</c><01:09:32.920><c> call</c>"
+    },
+    {
+      "start": 4173.03,
+      "duration": 0.0,
+      "text": "also that's another reason why we call"
+    },
+    {
+      "start": 4173.04,
+      "duration": 0.0,
+      "text": "also that's another reason why we call it<01:09:33.159><c> post</c><01:09:33.400><c> training</c><01:09:33.679><c> is</c><01:09:33.799><c> that</c><01:09:33.920><c> we</c><01:09:34.040><c> use</c>"
+    },
+    {
+      "start": 4174.229,
+      "duration": 0.0,
+      "text": "it post training is that we use"
+    },
+    {
+      "start": 4174.239,
+      "duration": 0.0,
+      "text": "it post training is that we use different<01:09:34.480><c> type</c><01:09:34.640><c> of</c><01:09:34.759><c> hyper</c><01:09:35.040><c> parameters</c><01:09:35.640><c> so</c>"
+    },
+    {
+      "start": 4175.709,
+      "duration": 0.0,
+      "text": "different type of hyper parameters so"
+    },
+    {
+      "start": 4175.719,
+      "duration": 0.0,
+      "text": "different type of hyper parameters so you<01:09:35.839><c> know</c><01:09:36.000><c> I</c><01:09:36.120><c> told</c><01:09:36.319><c> you</c><01:09:36.600><c> basically</c><01:09:36.920><c> at</c><01:09:37.000><c> the</c><01:09:37.080><c> end</c>"
+    },
+    {
+      "start": 4177.149,
+      "duration": 0.0,
+      "text": "you know I told you basically at the end"
+    },
+    {
+      "start": 4177.159,
+      "duration": 0.0,
+      "text": "you know I told you basically at the end of<01:09:37.279><c> pre</c><01:09:37.440><c> training</c><01:09:37.759><c> you</c><01:09:37.960><c> essentially</c><01:09:38.319><c> end</c><01:09:38.480><c> up</c>"
+    },
+    {
+      "start": 4178.59,
+      "duration": 0.0,
+      "text": "of pre training you essentially end up"
+    },
+    {
+      "start": 4178.6,
+      "duration": 0.0,
+      "text": "of pre training you essentially end up with<01:09:38.719><c> a</c><01:09:38.799><c> learning</c><01:09:39.120><c> rate</c><01:09:39.279><c> of</c><01:09:39.440><c> zero</c><01:09:40.239><c> and</c><01:09:40.400><c> here</c>"
+    },
+    {
+      "start": 4180.51,
+      "duration": 0.0,
+      "text": "with a learning rate of zero and here"
+    },
+    {
+      "start": 4180.52,
+      "duration": 0.0,
+      "text": "with a learning rate of zero and here you're<01:09:40.640><c> going</c><01:09:40.759><c> to</c><01:09:40.880><c> increase</c><01:09:41.199><c> your</c><01:09:41.359><c> learning</c>"
+    },
+    {
+      "start": 4181.669,
+      "duration": 0.0,
+      "text": "you're going to increase your learning"
+    },
+    {
+      "start": 4181.679,
+      "duration": 0.0,
+      "text": "you're going to increase your learning rate<01:09:42.080><c> so</c><01:09:42.279><c> like</c><01:09:42.400><c> 1</c><01:09:42.560><c> eus</c><01:09:43.000><c> 5</c><01:09:43.199><c> one</c><01:09:43.359><c> E</c><01:09:43.920><c> Yeah</c><01:09:44.319><c> and</c><01:09:44.560><c> and</c>"
+    },
+    {
+      "start": 4184.829,
+      "duration": 0.0,
+      "text": "rate so like 1 eus 5 one E Yeah and and"
+    },
+    {
+      "start": 4184.839,
+      "duration": 0.0,
+      "text": "rate so like 1 eus 5 one E Yeah and and so<01:09:45.839><c> um</c><01:09:46.279><c> the</c><01:09:46.480><c> weight</c><01:09:46.759><c> that</c><01:09:46.880><c> you</c><01:09:47.040><c> give</c><01:09:47.199><c> to</c><01:09:47.440><c> them</c>"
+    },
+    {
+      "start": 4187.95,
+      "duration": 0.0,
+      "text": "so um the weight that you give to them"
+    },
+    {
+      "start": 4187.96,
+      "duration": 0.0,
+      "text": "so um the weight that you give to them is<01:09:48.159><c> actually</c>"
+    },
+    {
+      "start": 4189.309,
+      "duration": 0.0,
+      "text": "is actually"
+    },
+    {
+      "start": 4189.319,
+      "duration": 0.0,
+      "text": "is actually different"
+    },
+    {
+      "start": 4191.87,
+      "duration": 0.0,
+      "text": "different"
+    },
+    {
+      "start": 4191.88,
+      "duration": 0.0,
+      "text": "different um<01:09:52.960><c> okay</c><01:09:53.960><c> uh</c><01:09:54.159><c> Second</c><01:09:54.480><c> Step</c><01:09:54.840><c> or</c><01:09:55.040><c> second</c><01:09:55.560><c> part</c><01:09:55.960><c> of</c>"
+    },
+    {
+      "start": 4196.189,
+      "duration": 0.0,
+      "text": "um okay uh Second Step or second part of"
+    },
+    {
+      "start": 4196.199,
+      "duration": 0.0,
+      "text": "um okay uh Second Step or second part of this<01:09:56.719><c> post</c><01:09:57.000><c> training</c><01:09:57.840><c> um</c><01:09:58.080><c> is</c><01:09:58.280><c> what</c><01:09:58.400><c> we</c><01:09:58.560><c> call</c>"
+    },
+    {
+      "start": 4199.11,
+      "duration": 0.0,
+      "text": "this post training um is what we call"
+    },
+    {
+      "start": 4199.12,
+      "duration": 0.0,
+      "text": "this post training um is what we call reinforcement<01:09:59.760><c> learning</c><01:10:00.040><c> from</c><01:10:00.280><c> Human</c>"
+    },
+    {
+      "start": 4200.63,
+      "duration": 0.0,
+      "text": "reinforcement learning from Human"
+    },
+    {
+      "start": 4200.64,
+      "duration": 0.0,
+      "text": "reinforcement learning from Human feedback<01:10:01.120><c> or</c><01:10:01.440><c> rhf</c><01:10:02.440><c> uh</c><01:10:02.560><c> some</c><01:10:02.760><c> of</c><01:10:02.880><c> you</c><01:10:03.040><c> might</c>"
+    },
+    {
+      "start": 4203.229,
+      "duration": 0.0,
+      "text": "feedback or rhf uh some of you might"
+    },
+    {
+      "start": 4203.239,
+      "duration": 0.0,
+      "text": "feedback or rhf uh some of you might have<01:10:03.440><c> heard</c><01:10:03.640><c> of</c><01:10:03.800><c> that</c><01:10:04.719><c> um</c><01:10:05.520><c> the</c><01:10:05.760><c> idea</c><01:10:06.120><c> is</c><01:10:06.239><c> that</c>"
+    },
+    {
+      "start": 4206.43,
+      "duration": 0.0,
+      "text": "have heard of that um the idea is that"
+    },
+    {
+      "start": 4206.44,
+      "duration": 0.0,
+      "text": "have heard of that um the idea is that sft<01:10:06.920><c> has</c><01:10:07.040><c> a</c><01:10:07.280><c> problem</c><01:10:07.960><c> namely</c><01:10:08.520><c> that</c><01:10:08.960><c> uh</c><01:10:09.040><c> you</c><01:10:09.159><c> do</c>"
+    },
+    {
+      "start": 4209.43,
+      "duration": 0.0,
+      "text": "sft has a problem namely that uh you do"
+    },
+    {
+      "start": 4209.44,
+      "duration": 0.0,
+      "text": "sft has a problem namely that uh you do behavioral<01:10:10.239><c> cloning</c><01:10:10.840><c> which</c><01:10:10.960><c> means</c><01:10:11.199><c> that</c><01:10:11.360><c> you</c>"
+    },
+    {
+      "start": 4211.47,
+      "duration": 0.0,
+      "text": "behavioral cloning which means that you"
+    },
+    {
+      "start": 4211.48,
+      "duration": 0.0,
+      "text": "behavioral cloning which means that you just<01:10:11.640><c> try</c><01:10:11.840><c> to</c><01:10:12.040><c> clone</c><01:10:12.760><c> what</c><01:10:12.960><c> the</c><01:10:13.159><c> humans</c><01:10:13.800><c> would</c>"
+    },
+    {
+      "start": 4214.07,
+      "duration": 0.0,
+      "text": "just try to clone what the humans would"
+    },
+    {
+      "start": 4214.08,
+      "duration": 0.0,
+      "text": "just try to clone what the humans would say<01:10:14.679><c> and</c><01:10:14.800><c> that</c><01:10:15.000><c> had</c><01:10:15.520><c> that</c><01:10:15.640><c> has</c><01:10:15.760><c> many</c><01:10:16.000><c> issues</c>"
+    },
+    {
+      "start": 4216.669,
+      "duration": 0.0,
+      "text": "say and that had that has many issues"
+    },
+    {
+      "start": 4216.679,
+      "duration": 0.0,
+      "text": "say and that had that has many issues one<01:10:16.840><c> of</c><01:10:17.040><c> them</c><01:10:17.239><c> is</c><01:10:17.360><c> that</c><01:10:17.480><c> you're</c><01:10:17.640><c> bound</c><01:10:17.920><c> by</c>"
+    },
+    {
+      "start": 4218.07,
+      "duration": 0.0,
+      "text": "one of them is that you're bound by"
+    },
+    {
+      "start": 4218.08,
+      "duration": 0.0,
+      "text": "one of them is that you're bound by human<01:10:18.520><c> abilities</c><01:10:19.520><c> so</c><01:10:20.280><c> if</c><01:10:21.239><c> um</c><01:10:22.239><c> like</c><01:10:22.520><c> humans</c>"
+    },
+    {
+      "start": 4223.55,
+      "duration": 0.0,
+      "text": "human abilities so if um like humans"
+    },
+    {
+      "start": 4223.56,
+      "duration": 0.0,
+      "text": "human abilities so if um like humans actually<01:10:24.560><c> humans</c><01:10:25.320><c> won't</c><01:10:25.600><c> generate</c><01:10:26.000><c> the</c>"
+    },
+    {
+      "start": 4226.149,
+      "duration": 0.0,
+      "text": "actually humans won't generate the"
+    },
+    {
+      "start": 4226.159,
+      "duration": 0.0,
+      "text": "actually humans won't generate the things<01:10:26.400><c> that</c><01:10:26.560><c> they</c><01:10:26.719><c> think</c><01:10:26.880><c> is</c><01:10:27.040><c> actually</c><01:10:27.280><c> the</c>"
+    },
+    {
+      "start": 4227.39,
+      "duration": 0.0,
+      "text": "things that they think is actually the"
+    },
+    {
+      "start": 4227.4,
+      "duration": 0.0,
+      "text": "things that they think is actually the best<01:10:27.560><c> thing</c><01:10:27.719><c> to</c><01:10:27.840><c> generate</c><01:10:28.600><c> so</c><01:10:28.920><c> if</c><01:10:29.040><c> you</c><01:10:29.280><c> ask</c><01:10:29.480><c> me</c>"
+    },
+    {
+      "start": 4229.63,
+      "duration": 0.0,
+      "text": "best thing to generate so if you ask me"
+    },
+    {
+      "start": 4229.64,
+      "duration": 0.0,
+      "text": "best thing to generate so if you ask me to<01:10:29.800><c> write</c><01:10:29.960><c> a</c><01:10:30.120><c> book</c><01:10:30.760><c> I</c><01:10:30.840><c> mean</c><01:10:31.000><c> I</c><01:10:31.080><c> can</c><01:10:31.199><c> definitely</c>"
+    },
+    {
+      "start": 4231.51,
+      "duration": 0.0,
+      "text": "to write a book I mean I can definitely"
+    },
+    {
+      "start": 4231.52,
+      "duration": 0.0,
+      "text": "to write a book I mean I can definitely enjoy<01:10:31.800><c> a</c><01:10:31.960><c> book</c><01:10:32.280><c> I</c><01:10:32.360><c> can</c><01:10:32.520><c> probably</c><01:10:32.760><c> say</c><01:10:32.960><c> one</c><01:10:33.120><c> book</c>"
+    },
+    {
+      "start": 4233.31,
+      "duration": 0.0,
+      "text": "enjoy a book I can probably say one book"
+    },
+    {
+      "start": 4233.32,
+      "duration": 0.0,
+      "text": "enjoy a book I can probably say one book is<01:10:33.440><c> better</c><01:10:33.640><c> than</c><01:10:33.800><c> another</c><01:10:34.640><c> but</c><01:10:34.760><c> I'm</c>"
+    },
+    {
+      "start": 4234.91,
+      "duration": 0.0,
+      "text": "is better than another but I'm"
+    },
+    {
+      "start": 4234.92,
+      "duration": 0.0,
+      "text": "is better than another but I'm definitely<01:10:35.159><c> not</c><01:10:35.280><c> going</c><01:10:35.360><c> to</c><01:10:35.480><c> be</c><01:10:35.640><c> as</c><01:10:35.760><c> good</c><01:10:35.920><c> as</c>"
+    },
+    {
+      "start": 4236.07,
+      "duration": 0.0,
+      "text": "definitely not going to be as good as"
+    },
+    {
+      "start": 4236.08,
+      "duration": 0.0,
+      "text": "definitely not going to be as good as writing<01:10:36.360><c> the</c><01:10:36.480><c> book</c><01:10:36.640><c> that</c><01:10:36.760><c> I</c><01:10:36.880><c> want</c><01:10:37.040><c> to</c><01:10:37.239><c> read</c><01:10:38.000><c> uh</c>"
+    },
+    {
+      "start": 4238.11,
+      "duration": 0.0,
+      "text": "writing the book that I want to read uh"
+    },
+    {
+      "start": 4238.12,
+      "duration": 0.0,
+      "text": "writing the book that I want to read uh so<01:10:38.239><c> you're</c><01:10:38.400><c> going</c><01:10:38.480><c> to</c><01:10:38.600><c> be</c><01:10:38.719><c> bound</c><01:10:39.040><c> by</c><01:10:39.159><c> the</c><01:10:39.280><c> human</c>"
+    },
+    {
+      "start": 4239.55,
+      "duration": 0.0,
+      "text": "so you're going to be bound by the human"
+    },
+    {
+      "start": 4239.56,
+      "duration": 0.0,
+      "text": "so you're going to be bound by the human ability<01:10:39.880><c> to</c><01:10:40.000><c> generate</c><01:10:40.400><c> things</c><01:10:40.719><c> even</c><01:10:40.960><c> though</c>"
+    },
+    {
+      "start": 4241.11,
+      "duration": 0.0,
+      "text": "ability to generate things even though"
+    },
+    {
+      "start": 4241.12,
+      "duration": 0.0,
+      "text": "ability to generate things even though the<01:10:41.280><c> humans</c><01:10:41.560><c> might</c><01:10:41.719><c> be</c><01:10:41.840><c> better</c><01:10:42.080><c> at</c>"
+    },
+    {
+      "start": 4242.229,
+      "duration": 0.0,
+      "text": "the humans might be better at"
+    },
+    {
+      "start": 4242.239,
+      "duration": 0.0,
+      "text": "the humans might be better at distinguishing<01:10:42.880><c> between</c><01:10:43.199><c> things</c><01:10:43.800><c> that's</c><01:10:44.000><c> one</c>"
+    },
+    {
+      "start": 4244.189,
+      "duration": 0.0,
+      "text": "distinguishing between things that's one"
+    },
+    {
+      "start": 4244.199,
+      "duration": 0.0,
+      "text": "distinguishing between things that's one issue<01:10:44.880><c> issue</c><01:10:45.120><c> number</c><01:10:45.360><c> two</c><01:10:46.280><c> uh</c><01:10:46.400><c> I</c><01:10:46.520><c> find</c><01:10:46.679><c> that</c>"
+    },
+    {
+      "start": 4246.87,
+      "duration": 0.0,
+      "text": "issue issue number two uh I find that"
+    },
+    {
+      "start": 4246.88,
+      "duration": 0.0,
+      "text": "issue issue number two uh I find that actually<01:10:47.120><c> pretty</c><01:10:47.320><c> interesting</c><01:10:47.800><c> is</c><01:10:48.000><c> that</c><01:10:48.600><c> it</c>"
+    },
+    {
+      "start": 4248.87,
+      "duration": 0.0,
+      "text": "actually pretty interesting is that it"
+    },
+    {
+      "start": 4248.88,
+      "duration": 0.0,
+      "text": "actually pretty interesting is that it might<01:10:49.159><c> if</c><01:10:49.280><c> you</c><01:10:49.400><c> ever</c><01:10:49.600><c> heard</c><01:10:49.760><c> of</c><01:10:49.920><c> the</c><01:10:50.000><c> word</c>"
+    },
+    {
+      "start": 4250.27,
+      "duration": 0.0,
+      "text": "might if you ever heard of the word"
+    },
+    {
+      "start": 4250.28,
+      "duration": 0.0,
+      "text": "might if you ever heard of the word hallucination<01:10:50.960><c> so</c><01:10:51.159><c> this</c><01:10:51.239><c> is</c><01:10:51.480><c> llms</c><01:10:52.080><c> generating</c>"
+    },
+    {
+      "start": 4253.07,
+      "duration": 0.0,
+      "text": "hallucination so this is llms generating"
+    },
+    {
+      "start": 4253.08,
+      "duration": 0.0,
+      "text": "hallucination so this is llms generating F<01:10:53.440><c> like</c><01:10:53.840><c> false</c><01:10:54.239><c> information</c>"
+    },
+    {
+      "start": 4256.149,
+      "duration": 0.0,
+      "text": "F like false information"
+    },
+    {
+      "start": 4256.159,
+      "duration": 0.0,
+      "text": "F like false information hallucination<01:10:57.159><c> might</c><01:10:57.480><c> these</c><01:10:57.679><c> people</c><01:10:57.960><c> have</c><01:10:58.520><c> um</c>"
+    },
+    {
+      "start": 4258.83,
+      "duration": 0.0,
+      "text": "hallucination might these people have um"
+    },
+    {
+      "start": 4258.84,
+      "duration": 0.0,
+      "text": "hallucination might these people have um hypothesized<01:10:59.480><c> that</c><01:10:59.679><c> that</c><01:10:59.840><c> can</c><01:11:00.080><c> come</c><01:11:00.320><c> from</c><01:11:00.560><c> the</c>"
+    },
+    {
+      "start": 4260.709,
+      "duration": 0.0,
+      "text": "hypothesized that that can come from the"
+    },
+    {
+      "start": 4260.719,
+      "duration": 0.0,
+      "text": "hypothesized that that can come from the supervised<01:11:01.199><c> fine</c><01:11:01.400><c> tuning</c><01:11:02.120><c> even</c><01:11:02.360><c> if</c><01:11:02.480><c> you</c><01:11:02.600><c> do</c>"
+    },
+    {
+      "start": 4262.79,
+      "duration": 0.0,
+      "text": "supervised fine tuning even if you do"
+    },
+    {
+      "start": 4262.8,
+      "duration": 0.0,
+      "text": "supervised fine tuning even if you do supervised<01:11:03.320><c> fine</c><01:11:03.520><c> tuning</c><01:11:04.239><c> on</c><01:11:04.840><c> data</c><01:11:05.159><c> that</c><01:11:05.280><c> is</c>"
+    },
+    {
+      "start": 4265.47,
+      "duration": 0.0,
+      "text": "supervised fine tuning on data that is"
+    },
+    {
+      "start": 4265.48,
+      "duration": 0.0,
+      "text": "supervised fine tuning on data that is correct<01:11:06.320><c> and</c><01:11:06.440><c> the</c><01:11:06.600><c> reason</c><01:11:06.960><c> why</c><01:11:07.159><c> that</c><01:11:07.360><c> is</c><01:11:08.000><c> is</c>"
+    },
+    {
+      "start": 4268.189,
+      "duration": 0.0,
+      "text": "correct and the reason why that is is"
+    },
+    {
+      "start": 4268.199,
+      "duration": 0.0,
+      "text": "correct and the reason why that is is that<01:11:08.600><c> if</c><01:11:09.440><c> uh</c><01:11:09.600><c> given</c><01:11:09.920><c> I</c><01:11:10.040><c> told</c><01:11:10.239><c> you</c><01:11:10.360><c> that</c>"
+    },
+    {
+      "start": 4270.51,
+      "duration": 0.0,
+      "text": "that if uh given I told you that"
+    },
+    {
+      "start": 4270.52,
+      "duration": 0.0,
+      "text": "that if uh given I told you that basically<01:11:10.960><c> sftt</c><01:11:11.640><c> is</c><01:11:11.840><c> with</c><01:11:12.199><c> very</c><01:11:12.400><c> little</c><01:11:12.679><c> data</c>"
+    },
+    {
+      "start": 4273.31,
+      "duration": 0.0,
+      "text": "basically sftt is with very little data"
+    },
+    {
+      "start": 4273.32,
+      "duration": 0.0,
+      "text": "basically sftt is with very little data and<01:11:13.440><c> it's</c><01:11:13.679><c> with</c><01:11:13.880><c> data</c><01:11:14.719><c> that</c><01:11:14.880><c> doesn't</c><01:11:15.320><c> the</c>"
+    },
+    {
+      "start": 4275.51,
+      "duration": 0.0,
+      "text": "and it's with data that doesn't the"
+    },
+    {
+      "start": 4275.52,
+      "duration": 0.0,
+      "text": "and it's with data that doesn't the model<01:11:15.840><c> doesn't</c><01:11:16.080><c> learn</c><01:11:16.360><c> anything</c><01:11:16.719><c> new</c><01:11:17.480><c> so</c><01:11:17.760><c> what</c>"
+    },
+    {
+      "start": 4277.91,
+      "duration": 0.0,
+      "text": "model doesn't learn anything new so what"
+    },
+    {
+      "start": 4277.92,
+      "duration": 0.0,
+      "text": "model doesn't learn anything new so what if<01:11:18.239><c> the</c><01:11:18.440><c> human</c><01:11:18.800><c> gives</c><01:11:19.000><c> an</c><01:11:19.480><c> answer</c><01:11:20.480><c> that</c><01:11:20.640><c> the</c>"
+    },
+    {
+      "start": 4280.79,
+      "duration": 0.0,
+      "text": "if the human gives an answer that the"
+    },
+    {
+      "start": 4280.8,
+      "duration": 0.0,
+      "text": "if the human gives an answer that the model<01:11:21.159><c> didn't</c><01:11:21.440><c> know</c><01:11:22.000><c> was</c><01:11:22.480><c> true</c><01:11:23.480><c> from</c><01:11:23.679><c> the</c>"
+    },
+    {
+      "start": 4283.83,
+      "duration": 0.0,
+      "text": "model didn't know was true from the"
+    },
+    {
+      "start": 4283.84,
+      "duration": 0.0,
+      "text": "model didn't know was true from the model<01:11:24.199><c> perspective</c><01:11:24.960><c> you</c><01:11:25.400><c> the</c><01:11:25.560><c> human</c>"
+    },
+    {
+      "start": 4285.87,
+      "duration": 0.0,
+      "text": "model perspective you the human"
+    },
+    {
+      "start": 4285.88,
+      "duration": 0.0,
+      "text": "model perspective you the human basically<01:11:26.199><c> is</c><01:11:26.320><c> telling</c><01:11:26.760><c> the</c><01:11:27.000><c> the</c><01:11:27.120><c> model</c><01:11:28.120><c> uh</c>"
+    },
+    {
+      "start": 4288.43,
+      "duration": 0.0,
+      "text": "basically is telling the the model uh"
+    },
+    {
+      "start": 4288.44,
+      "duration": 0.0,
+      "text": "basically is telling the the model uh generate<01:11:28.880><c> this</c><01:11:29.080><c> thing</c><01:11:29.360><c> that</c><01:11:29.520><c> seems</c><01:11:30.040><c> plausible</c>"
+    },
+    {
+      "start": 4291.03,
+      "duration": 0.0,
+      "text": "generate this thing that seems plausible"
+    },
+    {
+      "start": 4291.04,
+      "duration": 0.0,
+      "text": "generate this thing that seems plausible but<01:11:31.280><c> actually</c><01:11:31.520><c> have</c><01:11:31.640><c> no</c><01:11:31.800><c> idea</c><01:11:32.080><c> if</c><01:11:32.159><c> it's</c><01:11:32.360><c> true</c>"
+    },
+    {
+      "start": 4292.55,
+      "duration": 0.0,
+      "text": "but actually have no idea if it's true"
+    },
+    {
+      "start": 4292.56,
+      "duration": 0.0,
+      "text": "but actually have no idea if it's true or<01:11:32.719><c> not</c><01:11:33.679><c> um</c><01:11:34.520><c> so</c><01:11:34.800><c> just</c><01:11:34.920><c> to</c><01:11:35.080><c> give</c><01:11:35.199><c> you</c><01:11:35.280><c> a</c><01:11:35.400><c> very</c>"
+    },
+    {
+      "start": 4295.55,
+      "duration": 0.0,
+      "text": "or not um so just to give you a very"
+    },
+    {
+      "start": 4295.56,
+      "duration": 0.0,
+      "text": "or not um so just to give you a very concrete<01:11:35.960><c> example</c><01:11:36.560><c> if</c><01:11:36.679><c> we</c><01:11:36.840><c> go</c><01:11:37.000><c> back</c><01:11:37.159><c> to</c><01:11:37.360><c> this</c>"
+    },
+    {
+      "start": 4297.709,
+      "duration": 0.0,
+      "text": "concrete example if we go back to this"
+    },
+    {
+      "start": 4297.719,
+      "duration": 0.0,
+      "text": "concrete example if we go back to this uh<01:11:37.880><c> monopsony</c><01:11:38.560><c> example</c><01:11:39.000><c> can</c><01:11:39.120><c> you</c><01:11:39.280><c> write</c><01:11:39.480><c> blah</c>"
+    },
+    {
+      "start": 4299.669,
+      "duration": 0.0,
+      "text": "uh monopsony example can you write blah"
+    },
+    {
+      "start": 4299.679,
+      "duration": 0.0,
+      "text": "uh monopsony example can you write blah blah<01:11:39.840><c> blah</c><01:11:40.320><c> about</c><01:11:40.639><c> monopsony</c><01:11:41.639><c> uh</c><01:11:41.800><c> imagine</c>"
+    },
+    {
+      "start": 4302.07,
+      "duration": 0.0,
+      "text": "blah blah about monopsony uh imagine"
+    },
+    {
+      "start": 4302.08,
+      "duration": 0.0,
+      "text": "blah blah about monopsony uh imagine that<01:11:42.239><c> a</c><01:11:42.440><c> human</c><01:11:43.159><c> uh</c><01:11:43.320><c> wrote</c><01:11:43.560><c> a</c><01:11:43.760><c> reference</c><01:11:44.440><c> on</c>"
+    },
+    {
+      "start": 4304.75,
+      "duration": 0.0,
+      "text": "that a human uh wrote a reference on"
+    },
+    {
+      "start": 4304.76,
+      "duration": 0.0,
+      "text": "that a human uh wrote a reference on this<01:11:44.960><c> type</c><01:11:45.159><c> of</c><01:11:45.320><c> book</c><01:11:46.239><c> um</c><01:11:46.679><c> and</c><01:11:46.920><c> that</c><01:11:47.080><c> book</c><01:11:47.280><c> might</c>"
+    },
+    {
+      "start": 4307.51,
+      "duration": 0.0,
+      "text": "this type of book um and that book might"
+    },
+    {
+      "start": 4307.52,
+      "duration": 0.0,
+      "text": "this type of book um and that book might exist<01:11:47.840><c> that</c><01:11:47.960><c> might</c><01:11:48.080><c> be</c><01:11:48.199><c> a</c><01:11:48.320><c> correct</c><01:11:48.679><c> reference</c>"
+    },
+    {
+      "start": 4309.43,
+      "duration": 0.0,
+      "text": "exist that might be a correct reference"
+    },
+    {
+      "start": 4309.44,
+      "duration": 0.0,
+      "text": "exist that might be a correct reference but<01:11:49.600><c> what</c><01:11:49.719><c> if</c><01:11:49.840><c> the</c><01:11:49.960><c> llm</c><01:11:50.440><c> never</c><01:11:50.679><c> saw</c><01:11:51.080><c> this</c>"
+    },
+    {
+      "start": 4311.27,
+      "duration": 0.0,
+      "text": "but what if the llm never saw this"
+    },
+    {
+      "start": 4311.28,
+      "duration": 0.0,
+      "text": "but what if the llm never saw this reference<01:11:51.679><c> during</c><01:11:51.960><c> pre-training</c><01:11:52.760><c> then</c><01:11:52.880><c> it</c>"
+    },
+    {
+      "start": 4312.99,
+      "duration": 0.0,
+      "text": "reference during pre-training then it"
+    },
+    {
+      "start": 4313.0,
+      "duration": 0.0,
+      "text": "reference during pre-training then it doesn't<01:11:53.280><c> know</c><01:11:53.480><c> that</c><01:11:53.600><c> it's</c><01:11:53.679><c> a</c><01:11:53.800><c> correct</c>"
+    },
+    {
+      "start": 4314.07,
+      "duration": 0.0,
+      "text": "doesn't know that it's a correct"
+    },
+    {
+      "start": 4314.08,
+      "duration": 0.0,
+      "text": "doesn't know that it's a correct reference<01:11:54.639><c> so</c><01:11:54.800><c> really</c><01:11:55.000><c> what</c><01:11:55.080><c> you</c><01:11:55.440><c> tell</c><01:11:55.639><c> the</c>"
+    },
+    {
+      "start": 4315.79,
+      "duration": 0.0,
+      "text": "reference so really what you tell the"
+    },
+    {
+      "start": 4315.8,
+      "duration": 0.0,
+      "text": "reference so really what you tell the model<01:11:56.400><c> is</c><01:11:56.600><c> to</c><01:11:57.000><c> generate</c><01:11:57.400><c> or</c><01:11:57.639><c> make</c><01:11:57.840><c> up</c><01:11:58.040><c> some</c>"
+    },
+    {
+      "start": 4318.229,
+      "duration": 0.0,
+      "text": "model is to generate or make up some"
+    },
+    {
+      "start": 4318.239,
+      "duration": 0.0,
+      "text": "model is to generate or make up some plausibly<01:11:58.760><c> sounding</c><01:11:59.520><c> reference</c><01:12:00.520><c> um</c><01:12:01.159><c> rather</c>"
+    },
+    {
+      "start": 4321.47,
+      "duration": 0.0,
+      "text": "plausibly sounding reference um rather"
+    },
+    {
+      "start": 4321.48,
+      "duration": 0.0,
+      "text": "plausibly sounding reference um rather than<01:12:01.760><c> actually</c><01:12:02.679><c> tell</c><01:12:02.920><c> the</c><01:12:03.080><c> real</c><01:12:03.360><c> reference</c>"
+    },
+    {
+      "start": 4323.669,
+      "duration": 0.0,
+      "text": "than actually tell the real reference"
+    },
+    {
+      "start": 4323.679,
+      "duration": 0.0,
+      "text": "than actually tell the real reference that<01:12:03.800><c> it</c><01:12:03.920><c> saw</c><01:12:04.120><c> during</c><01:12:04.440><c> pre-training</c><01:12:05.440><c> uh</c><01:12:05.600><c> so</c>"
+    },
+    {
+      "start": 4326.189,
+      "duration": 0.0,
+      "text": "that it saw during pre-training uh so"
+    },
+    {
+      "start": 4326.199,
+      "duration": 0.0,
+      "text": "that it saw during pre-training uh so hallucination<01:12:07.040><c> might</c><01:12:07.880><c> be</c><01:12:08.840><c> um</c><01:12:09.600><c> uh</c><01:12:09.920><c> a</c><01:12:10.159><c> re</c><01:12:10.480><c> like</c>"
+    },
+    {
+      "start": 4330.669,
+      "duration": 0.0,
+      "text": "hallucination might be um uh a re like"
+    },
+    {
+      "start": 4330.679,
+      "duration": 0.0,
+      "text": "hallucination might be um uh a re like might<01:12:10.840><c> be</c><01:12:11.000><c> caused</c><01:12:11.400><c> by</c><01:12:11.560><c> this</c><01:12:11.719><c> sft</c><01:12:12.520><c> that's</c>"
+    },
+    {
+      "start": 4332.95,
+      "duration": 0.0,
+      "text": "might be caused by this sft that's"
+    },
+    {
+      "start": 4332.96,
+      "duration": 0.0,
+      "text": "might be caused by this sft that's problem<01:12:13.239><c> number</c><01:12:13.440><c> two</c><01:12:14.199><c> does</c><01:12:14.360><c> that</c><01:12:14.480><c> all</c><01:12:14.639><c> make</c>"
+    },
+    {
+      "start": 4334.79,
+      "duration": 0.0,
+      "text": "problem number two does that all make"
+    },
+    {
+      "start": 4334.8,
+      "duration": 0.0,
+      "text": "problem number two does that all make sense<01:12:15.800><c> great</c><01:12:16.560><c> problem</c><01:12:16.840><c> number</c><01:12:17.080><c> three</c><01:12:17.639><c> price</c>"
+    },
+    {
+      "start": 4338.47,
+      "duration": 0.0,
+      "text": "sense great problem number three price"
+    },
+    {
+      "start": 4338.48,
+      "duration": 0.0,
+      "text": "sense great problem number three price generating<01:12:19.000><c> the</c><01:12:19.239><c> ideal</c><01:12:19.800><c> answers</c><01:12:20.800><c> is</c><01:12:21.040><c> very</c>"
+    },
+    {
+      "start": 4341.229,
+      "duration": 0.0,
+      "text": "generating the ideal answers is very"
+    },
+    {
+      "start": 4341.239,
+      "duration": 0.0,
+      "text": "generating the ideal answers is very pricey<01:12:21.679><c> and</c><01:12:21.800><c> that</c><01:12:21.960><c> comes</c><01:12:22.199><c> back</c><01:12:22.320><c> to</c><01:12:22.440><c> your</c>"
+    },
+    {
+      "start": 4342.629,
+      "duration": 0.0,
+      "text": "pricey and that comes back to your"
+    },
+    {
+      "start": 4342.639,
+      "duration": 0.0,
+      "text": "pricey and that comes back to your question<01:12:23.440><c> um</c><01:12:23.719><c> of</c><01:12:23.960><c> like</c><01:12:24.400><c> humans</c><01:12:24.679><c> writing</c>"
+    },
+    {
+      "start": 4345.47,
+      "duration": 0.0,
+      "text": "question um of like humans writing"
+    },
+    {
+      "start": 4345.48,
+      "duration": 0.0,
+      "text": "question um of like humans writing answer<01:12:26.000><c> is</c><01:12:26.199><c> actually</c><01:12:26.440><c> pretty</c>"
+    },
+    {
+      "start": 4347.31,
+      "duration": 0.0,
+      "text": "answer is actually pretty"
+    },
+    {
+      "start": 4347.32,
+      "duration": 0.0,
+      "text": "answer is actually pretty expensive<01:12:28.320><c> um</c><01:12:28.520><c> so</c><01:12:28.679><c> that's</c><01:12:28.840><c> where</c><01:12:29.000><c> rhf</c><01:12:29.600><c> comes</c>"
+    },
+    {
+      "start": 4349.79,
+      "duration": 0.0,
+      "text": "expensive um so that's where rhf comes"
+    },
+    {
+      "start": 4349.8,
+      "duration": 0.0,
+      "text": "expensive um so that's where rhf comes in<01:12:30.360><c> the</c><01:12:30.520><c> idea</c><01:12:30.840><c> is</c><01:12:31.000><c> that</c><01:12:31.239><c> instead</c><01:12:31.600><c> of</c><01:12:31.800><c> cloning</c>"
+    },
+    {
+      "start": 4352.27,
+      "duration": 0.0,
+      "text": "in the idea is that instead of cloning"
+    },
+    {
+      "start": 4352.28,
+      "duration": 0.0,
+      "text": "in the idea is that instead of cloning the<01:12:32.440><c> behaviors</c><01:12:33.360><c> of</c><01:12:33.560><c> humans</c><01:12:34.159><c> we're</c><01:12:34.320><c> going</c><01:12:34.400><c> to</c>"
+    },
+    {
+      "start": 4354.669,
+      "duration": 0.0,
+      "text": "the behaviors of humans we're going to"
+    },
+    {
+      "start": 4354.679,
+      "duration": 0.0,
+      "text": "the behaviors of humans we're going to maximize<01:12:35.320><c> human</c><01:12:35.920><c> preference</c><01:12:36.920><c> um</c><01:12:37.280><c> and</c><01:12:37.400><c> the</c><01:12:37.520><c> way</c>"
+    },
+    {
+      "start": 4357.629,
+      "duration": 0.0,
+      "text": "maximize human preference um and the way"
+    },
+    {
+      "start": 4357.639,
+      "duration": 0.0,
+      "text": "maximize human preference um and the way we're<01:12:37.800><c> going</c><01:12:37.880><c> to</c><01:12:38.040><c> do</c><01:12:38.239><c> that</c><01:12:38.440><c> so</c><01:12:38.639><c> the</c><01:12:38.800><c> pipeline</c>"
+    },
+    {
+      "start": 4359.629,
+      "duration": 0.0,
+      "text": "we're going to do that so the pipeline"
+    },
+    {
+      "start": 4359.639,
+      "duration": 0.0,
+      "text": "we're going to do that so the pipeline is<01:12:39.800><c> that</c><01:12:40.000><c> for</c><01:12:40.639><c> a</c><01:12:40.840><c> certain</c><01:12:41.320><c> for</c><01:12:41.480><c> every</c>"
+    },
+    {
+      "start": 4361.669,
+      "duration": 0.0,
+      "text": "is that for a certain for every"
+    },
+    {
+      "start": 4361.679,
+      "duration": 0.0,
+      "text": "is that for a certain for every instruction<01:12:42.280><c> you're</c><01:12:42.400><c> going</c><01:12:42.520><c> to</c><01:12:42.719><c> ask</c><01:12:42.880><c> a</c><01:12:43.040><c> model</c>"
+    },
+    {
+      "start": 4363.27,
+      "duration": 0.0,
+      "text": "instruction you're going to ask a model"
+    },
+    {
+      "start": 4363.28,
+      "duration": 0.0,
+      "text": "instruction you're going to ask a model to<01:12:43.440><c> generate</c><01:12:43.920><c> two</c><01:12:44.960><c> answers</c><01:12:45.960><c> um</c><01:12:46.520><c> and</c><01:12:46.719><c> usually</c>"
+    },
+    {
+      "start": 4366.99,
+      "duration": 0.0,
+      "text": "to generate two answers um and usually"
+    },
+    {
+      "start": 4367.0,
+      "duration": 0.0,
+      "text": "to generate two answers um and usually use<01:12:47.159><c> a</c><01:12:47.400><c> pretty</c><01:12:47.639><c> good</c><01:12:47.800><c> model</c><01:12:48.080><c> so</c><01:12:48.199><c> you</c><01:12:48.280><c> usually</c>"
+    },
+    {
+      "start": 4368.59,
+      "duration": 0.0,
+      "text": "use a pretty good model so you usually"
+    },
+    {
+      "start": 4368.6,
+      "duration": 0.0,
+      "text": "use a pretty good model so you usually don't<01:12:48.760><c> use</c><01:12:48.920><c> an</c><01:12:49.080><c> LM</c><01:12:49.480><c> here</c><01:12:49.639><c> you</c><01:12:49.760><c> use</c><01:12:49.960><c> a</c><01:12:51.040><c> sft</c><01:12:52.040><c> uh</c>"
+    },
+    {
+      "start": 4372.149,
+      "duration": 0.0,
+      "text": "don't use an LM here you use a sft uh"
+    },
+    {
+      "start": 4372.159,
+      "duration": 0.0,
+      "text": "don't use an LM here you use a sft uh fine<01:12:52.400><c> tune</c><01:12:52.800><c> you</c><01:12:52.880><c> use</c><01:12:53.040><c> a</c><01:12:53.199><c> fine</c><01:12:53.440><c> tuned</c><01:12:53.760><c> llm</c>"
+    },
+    {
+      "start": 4374.43,
+      "duration": 0.0,
+      "text": "fine tune you use a fine tuned llm"
+    },
+    {
+      "start": 4374.44,
+      "duration": 0.0,
+      "text": "fine tune you use a fine tuned llm already<01:12:54.760><c> to</c><01:12:54.920><c> give</c><01:12:55.239><c> like</c><01:12:55.520><c> pretty</c><01:12:55.760><c> good</c><01:12:56.239><c> answers</c>"
+    },
+    {
+      "start": 4377.229,
+      "duration": 0.0,
+      "text": "already to give like pretty good answers"
+    },
+    {
+      "start": 4377.239,
+      "duration": 0.0,
+      "text": "already to give like pretty good answers and<01:12:57.400><c> then</c><01:12:57.560><c> you</c><01:12:57.840><c> ask</c><01:12:58.239><c> labelers</c><01:12:59.239><c> which</c><01:12:59.400><c> of</c><01:12:59.600><c> these</c>"
+    },
+    {
+      "start": 4379.79,
+      "duration": 0.0,
+      "text": "and then you ask labelers which of these"
+    },
+    {
+      "start": 4379.8,
+      "duration": 0.0,
+      "text": "and then you ask labelers which of these two<01:13:00.000><c> answers</c><01:13:00.360><c> was</c><01:13:00.560><c> better</c><01:13:01.199><c> so</c><01:13:01.440><c> select</c><01:13:01.760><c> the</c>"
+    },
+    {
+      "start": 4381.87,
+      "duration": 0.0,
+      "text": "two answers was better so select the"
+    },
+    {
+      "start": 4381.88,
+      "duration": 0.0,
+      "text": "two answers was better so select the preferred<01:13:02.280><c> one</c><01:13:03.000><c> and</c><01:13:03.360><c> then</c><01:13:03.840><c> with</c><01:13:04.080><c> different</c>"
+    },
+    {
+      "start": 4384.31,
+      "duration": 0.0,
+      "text": "preferred one and then with different"
+    },
+    {
+      "start": 4384.32,
+      "duration": 0.0,
+      "text": "preferred one and then with different type<01:13:04.520><c> of</c><01:13:04.639><c> algorithms</c><01:13:05.120><c> we're</c><01:13:05.239><c> going</c><01:13:05.320><c> to</c><01:13:05.400><c> talk</c>"
+    },
+    {
+      "start": 4385.55,
+      "duration": 0.0,
+      "text": "type of algorithms we're going to talk"
+    },
+    {
+      "start": 4385.56,
+      "duration": 0.0,
+      "text": "type of algorithms we're going to talk about<01:13:05.719><c> the</c><01:13:05.800><c> algorithms</c><01:13:06.800><c> um</c><01:13:07.000><c> you</c><01:13:07.159><c> just</c>"
+    },
+    {
+      "start": 4387.31,
+      "duration": 0.0,
+      "text": "about the algorithms um you just"
+    },
+    {
+      "start": 4387.32,
+      "duration": 0.0,
+      "text": "about the algorithms um you just fine-tune<01:13:07.840><c> the</c><01:13:07.960><c> model</c><01:13:08.280><c> to</c><01:13:08.440><c> generate</c><01:13:08.840><c> more</c><01:13:09.000><c> of</c>"
+    },
+    {
+      "start": 4389.11,
+      "duration": 0.0,
+      "text": "fine-tune the model to generate more of"
+    },
+    {
+      "start": 4389.12,
+      "duration": 0.0,
+      "text": "fine-tune the model to generate more of the<01:13:09.239><c> green</c><01:13:09.560><c> thing</c><01:13:09.960><c> than</c><01:13:10.120><c> the</c><01:13:10.280><c> red</c><01:13:10.480><c> thing</c><01:13:10.800><c> so</c>"
+    },
+    {
+      "start": 4390.95,
+      "duration": 0.0,
+      "text": "the green thing than the red thing so"
+    },
+    {
+      "start": 4390.96,
+      "duration": 0.0,
+      "text": "the green thing than the red thing so more<01:13:11.080><c> of</c><01:13:11.239><c> the</c><01:13:11.360><c> good</c><01:13:11.679><c> stuff</c><01:13:12.679><c> uh</c><01:13:12.840><c> so</c><01:13:13.000><c> now</c><01:13:13.159><c> the</c>"
+    },
+    {
+      "start": 4393.31,
+      "duration": 0.0,
+      "text": "more of the good stuff uh so now the"
+    },
+    {
+      "start": 4393.32,
+      "duration": 0.0,
+      "text": "more of the good stuff uh so now the question<01:13:13.520><c> is</c><01:13:13.760><c> how</c><01:13:14.000><c> and</c><01:13:14.080><c> we're</c><01:13:14.199><c> going</c><01:13:14.280><c> to</c><01:13:14.400><c> talk</c>"
+    },
+    {
+      "start": 4394.55,
+      "duration": 0.0,
+      "text": "question is how and we're going to talk"
+    },
+    {
+      "start": 4394.56,
+      "duration": 0.0,
+      "text": "question is how and we're going to talk about<01:13:14.760><c> that</c><01:13:14.960><c> right</c>"
+    },
+    {
+      "start": 4396.35,
+      "duration": 0.0,
+      "text": "about that right"
+    },
+    {
+      "start": 4396.36,
+      "duration": 0.0,
+      "text": "about that right now<01:13:17.360><c> so</c><01:13:17.800><c> there</c><01:13:17.960><c> are</c><01:13:18.400><c> two</c><01:13:18.840><c> ways</c><01:13:19.080><c> that</c><01:13:19.199><c> we're</c>"
+    },
+    {
+      "start": 4399.31,
+      "duration": 0.0,
+      "text": "now so there are two ways that we're"
+    },
+    {
+      "start": 4399.32,
+      "duration": 0.0,
+      "text": "now so there are two ways that we're going<01:13:19.400><c> to</c><01:13:19.520><c> talk</c><01:13:19.719><c> about</c><01:13:20.000><c> and</c><01:13:20.159><c> two</c><01:13:20.360><c> that</c><01:13:20.480><c> are</c>"
+    },
+    {
+      "start": 4400.87,
+      "duration": 0.0,
+      "text": "going to talk about and two that are"
+    },
+    {
+      "start": 4400.88,
+      "duration": 0.0,
+      "text": "going to talk about and two that are mainly<01:13:21.159><c> used</c><01:13:21.360><c> in</c><01:13:21.480><c> the</c><01:13:21.600><c> community</c><01:13:22.600><c> um</c><01:13:23.360><c> the</c>"
+    },
+    {
+      "start": 4403.51,
+      "duration": 0.0,
+      "text": "mainly used in the community um the"
+    },
+    {
+      "start": 4403.52,
+      "duration": 0.0,
+      "text": "mainly used in the community um the first<01:13:23.760><c> one</c><01:13:24.199><c> is</c><01:13:24.440><c> simply</c><01:13:24.719><c> the</c><01:13:24.840><c> idea</c><01:13:25.040><c> of</c><01:13:25.120><c> of</c><01:13:25.239><c> using</c>"
+    },
+    {
+      "start": 4405.43,
+      "duration": 0.0,
+      "text": "first one is simply the idea of of using"
+    },
+    {
+      "start": 4405.44,
+      "duration": 0.0,
+      "text": "first one is simply the idea of of using reinforcement<01:13:25.960><c> learning</c><01:13:26.360><c> so</c><01:13:26.520><c> hopefully</c><01:13:26.880><c> you</c>"
+    },
+    {
+      "start": 4406.95,
+      "duration": 0.0,
+      "text": "reinforcement learning so hopefully you"
+    },
+    {
+      "start": 4406.96,
+      "duration": 0.0,
+      "text": "reinforcement learning so hopefully you all<01:13:27.120><c> know</c><01:13:27.280><c> what</c><01:13:27.400><c> reinforcement</c><01:13:27.880><c> learning</c><01:13:28.239><c> is</c>"
+    },
+    {
+      "start": 4408.75,
+      "duration": 0.0,
+      "text": "all know what reinforcement learning is"
+    },
+    {
+      "start": 4408.76,
+      "duration": 0.0,
+      "text": "all know what reinforcement learning is now<01:13:29.679><c> um</c><01:13:30.320><c> so</c><01:13:30.920><c> when</c><01:13:31.120><c> you</c><01:13:31.960><c> think</c><01:13:32.199><c> about</c><01:13:32.360><c> using</c>"
+    },
+    {
+      "start": 4412.59,
+      "duration": 0.0,
+      "text": "now um so when you think about using"
+    },
+    {
+      "start": 4412.6,
+      "duration": 0.0,
+      "text": "now um so when you think about using reinforcement<01:13:33.120><c> learning</c><01:13:33.400><c> one</c><01:13:33.560><c> important</c>"
+    },
+    {
+      "start": 4413.87,
+      "duration": 0.0,
+      "text": "reinforcement learning one important"
+    },
+    {
+      "start": 4413.88,
+      "duration": 0.0,
+      "text": "reinforcement learning one important question<01:13:34.080><c> is</c><01:13:34.239><c> like</c><01:13:34.400><c> what</c><01:13:34.520><c> is</c><01:13:34.639><c> the</c><01:13:34.760><c> reward</c><01:13:35.080><c> that</c>"
+    },
+    {
+      "start": 4415.189,
+      "duration": 0.0,
+      "text": "question is like what is the reward that"
+    },
+    {
+      "start": 4415.199,
+      "duration": 0.0,
+      "text": "question is like what is the reward that we're<01:13:35.400><c> optimizing</c><01:13:36.400><c> uh</c><01:13:36.520><c> so</c><01:13:36.800><c> in</c><01:13:36.960><c> this</c><01:13:37.120><c> case</c>"
+    },
+    {
+      "start": 4417.31,
+      "duration": 0.0,
+      "text": "we're optimizing uh so in this case"
+    },
+    {
+      "start": 4417.32,
+      "duration": 0.0,
+      "text": "we're optimizing uh so in this case there<01:13:37.440><c> are</c><01:13:37.840><c> really</c><01:13:38.239><c> two</c><01:13:38.440><c> options</c><01:13:38.760><c> that</c><01:13:38.880><c> I</c>"
+    },
+    {
+      "start": 4418.95,
+      "duration": 0.0,
+      "text": "there are really two options that I"
+    },
+    {
+      "start": 4418.96,
+      "duration": 0.0,
+      "text": "there are really two options that I could<01:13:39.120><c> think</c><01:13:39.320><c> about</c><01:13:39.840><c> the</c><01:13:39.960><c> first</c><01:13:40.159><c> one</c><01:13:40.360><c> you</c>"
+    },
+    {
+      "start": 4420.47,
+      "duration": 0.0,
+      "text": "could think about the first one you"
+    },
+    {
+      "start": 4420.48,
+      "duration": 0.0,
+      "text": "could think about the first one you could<01:13:40.639><c> just</c><01:13:40.800><c> say</c><01:13:41.440><c> I'm</c><01:13:41.560><c> going</c><01:13:41.679><c> to</c><01:13:41.840><c> compare</c><01:13:42.199><c> the</c>"
+    },
+    {
+      "start": 4422.35,
+      "duration": 0.0,
+      "text": "could just say I'm going to compare the"
+    },
+    {
+      "start": 4422.36,
+      "duration": 0.0,
+      "text": "could just say I'm going to compare the output<01:13:42.719><c> generated</c><01:13:43.120><c> by</c><01:13:43.239><c> some</c><01:13:43.440><c> baseline</c><01:13:44.080><c> the</c>"
+    },
+    {
+      "start": 4424.229,
+      "duration": 0.0,
+      "text": "output generated by some baseline the"
+    },
+    {
+      "start": 4424.239,
+      "duration": 0.0,
+      "text": "output generated by some baseline the output<01:13:44.600><c> generated</c><01:13:45.040><c> by</c><01:13:45.199><c> my</c><01:13:45.360><c> model</c><01:13:46.320><c> U</c><01:13:46.440><c> and</c><01:13:46.560><c> I'm</c>"
+    },
+    {
+      "start": 4426.669,
+      "duration": 0.0,
+      "text": "output generated by my model U and I'm"
+    },
+    {
+      "start": 4426.679,
+      "duration": 0.0,
+      "text": "output generated by my model U and I'm just<01:13:46.800><c> going</c><01:13:46.880><c> to</c><01:13:47.040><c> ask</c><01:13:47.239><c> the</c><01:13:47.400><c> human</c><01:13:48.199><c> to</c><01:13:48.480><c> say</c><01:13:48.800><c> which</c>"
+    },
+    {
+      "start": 4428.95,
+      "duration": 0.0,
+      "text": "just going to ask the human to say which"
+    },
+    {
+      "start": 4428.96,
+      "duration": 0.0,
+      "text": "just going to ask the human to say which one<01:13:49.159><c> is</c><01:13:49.280><c> better</c><01:13:49.600><c> and</c><01:13:49.719><c> I'm</c><01:13:50.120><c> going</c><01:13:50.199><c> to</c><01:13:50.400><c> use</c><01:13:50.960><c> this</c>"
+    },
+    {
+      "start": 4431.07,
+      "duration": 0.0,
+      "text": "one is better and I'm going to use this"
+    },
+    {
+      "start": 4431.08,
+      "duration": 0.0,
+      "text": "one is better and I'm going to use this as<01:13:51.199><c> a</c><01:13:51.360><c> reward</c><01:13:51.760><c> so</c><01:13:51.920><c> if</c><01:13:52.040><c> I'm</c><01:13:52.199><c> better</c><01:13:52.440><c> than</c><01:13:52.560><c> the</c>"
+    },
+    {
+      "start": 4432.669,
+      "duration": 0.0,
+      "text": "as a reward so if I'm better than the"
+    },
+    {
+      "start": 4432.679,
+      "duration": 0.0,
+      "text": "as a reward so if I'm better than the Baseline<01:13:53.360><c> this</c><01:13:53.480><c> is</c><01:13:53.560><c> a</c><01:13:53.760><c> plus</c><01:13:53.960><c> one</c><01:13:54.239><c> if</c><01:13:54.360><c> not</c><01:13:54.480><c> it's</c>"
+    },
+    {
+      "start": 4434.59,
+      "duration": 0.0,
+      "text": "Baseline this is a plus one if not it's"
+    },
+    {
+      "start": 4434.6,
+      "duration": 0.0,
+      "text": "Baseline this is a plus one if not it's a<01:13:54.679><c> minus</c><01:13:54.920><c> one</c><01:13:55.159><c> one</c><01:13:55.800><c> uh</c><01:13:55.880><c> so</c><01:13:56.080><c> now</c><01:13:56.239><c> it's</c><01:13:56.520><c> binary</c>"
+    },
+    {
+      "start": 4436.91,
+      "duration": 0.0,
+      "text": "a minus one one uh so now it's binary"
+    },
+    {
+      "start": 4436.92,
+      "duration": 0.0,
+      "text": "a minus one one uh so now it's binary reward<01:13:57.480><c> the</c><01:13:57.639><c> problem</c><01:13:57.840><c> with</c><01:13:58.000><c> binary</c><01:13:58.320><c> reward</c><01:13:58.600><c> is</c>"
+    },
+    {
+      "start": 4438.709,
+      "duration": 0.0,
+      "text": "reward the problem with binary reward is"
+    },
+    {
+      "start": 4438.719,
+      "duration": 0.0,
+      "text": "reward the problem with binary reward is that<01:13:58.840><c> it's</c><01:13:59.000><c> very</c><01:13:59.159><c> sparse</c><01:13:59.840><c> and</c><01:13:59.920><c> you</c><01:14:00.000><c> don't</c><01:14:00.199><c> get</c>"
+    },
+    {
+      "start": 4440.35,
+      "duration": 0.0,
+      "text": "that it's very sparse and you don't get"
+    },
+    {
+      "start": 4440.36,
+      "duration": 0.0,
+      "text": "that it's very sparse and you don't get much<01:14:00.560><c> information</c><01:14:00.960><c> out</c><01:14:01.120><c> of</c><01:14:01.280><c> it</c><01:14:01.840><c> uh</c><01:14:02.040><c> like</c><01:14:02.320><c> maybe</c>"
+    },
+    {
+      "start": 4442.629,
+      "duration": 0.0,
+      "text": "much information out of it uh like maybe"
+    },
+    {
+      "start": 4442.639,
+      "duration": 0.0,
+      "text": "much information out of it uh like maybe your<01:14:02.800><c> answer</c><01:14:03.239><c> was</c><01:14:03.400><c> slightly</c><01:14:03.800><c> better</c><01:14:04.480><c> maybe</c><01:14:04.719><c> it</c>"
+    },
+    {
+      "start": 4444.83,
+      "duration": 0.0,
+      "text": "your answer was slightly better maybe it"
+    },
+    {
+      "start": 4444.84,
+      "duration": 0.0,
+      "text": "your answer was slightly better maybe it was<01:14:05.080><c> like</c><01:14:05.560><c> way</c><01:14:05.880><c> better</c><01:14:06.480><c> and</c><01:14:06.560><c> you</c><01:14:06.679><c> don't</c><01:14:06.880><c> really</c>"
+    },
+    {
+      "start": 4447.11,
+      "duration": 0.0,
+      "text": "was like way better and you don't really"
+    },
+    {
+      "start": 4447.12,
+      "duration": 0.0,
+      "text": "was like way better and you don't really know<01:14:07.719><c> from</c><01:14:08.120><c> this</c><01:14:08.679><c> um</c><01:14:09.199><c> how</c><01:14:09.360><c> much</c><01:14:09.560><c> better</c><01:14:09.800><c> it</c><01:14:10.159><c> was</c>"
+    },
+    {
+      "start": 4451.149,
+      "duration": 0.0,
+      "text": "know from this um how much better it was"
+    },
+    {
+      "start": 4451.159,
+      "duration": 0.0,
+      "text": "know from this um how much better it was so<01:14:11.320><c> option</c><01:14:11.600><c> two</c><01:14:12.199><c> is</c><01:14:12.320><c> that</c><01:14:12.480><c> you</c><01:14:12.560><c> can</c><01:14:12.679><c> train</c><01:14:13.040><c> what</c>"
+    },
+    {
+      "start": 4453.149,
+      "duration": 0.0,
+      "text": "so option two is that you can train what"
+    },
+    {
+      "start": 4453.159,
+      "duration": 0.0,
+      "text": "so option two is that you can train what we<01:14:13.280><c> call</c><01:14:13.400><c> a</c><01:14:13.560><c> reward</c><01:14:13.920><c> model</c><01:14:14.360><c> which</c><01:14:14.480><c> is</c><01:14:14.639><c> simply</c><01:14:15.120><c> a</c>"
+    },
+    {
+      "start": 4455.59,
+      "duration": 0.0,
+      "text": "we call a reward model which is simply a"
+    },
+    {
+      "start": 4455.6,
+      "duration": 0.0,
+      "text": "we call a reward model which is simply a classifier<01:14:16.600><c> uh</c><01:14:16.760><c> so</c><01:14:16.920><c> you</c><01:14:17.040><c> use</c><01:14:17.280><c> machine</c>"
+    },
+    {
+      "start": 4457.59,
+      "duration": 0.0,
+      "text": "classifier uh so you use machine"
+    },
+    {
+      "start": 4457.6,
+      "duration": 0.0,
+      "text": "classifier uh so you use machine learning<01:14:18.120><c> to</c><01:14:18.639><c> to</c><01:14:18.920><c> classify</c><01:14:19.920><c> how</c><01:14:20.199><c> much</c><01:14:20.480><c> better</c>"
+    },
+    {
+      "start": 4461.47,
+      "duration": 0.0,
+      "text": "learning to to classify how much better"
+    },
+    {
+      "start": 4461.48,
+      "duration": 0.0,
+      "text": "learning to to classify how much better uh<01:14:21.800><c> two</c><01:14:22.480><c> outputs</c><01:14:23.000><c> are</c><01:14:23.639><c> from</c><01:14:23.880><c> the</c><01:14:24.080><c> preference</c>"
+    },
+    {
+      "start": 4464.59,
+      "duration": 0.0,
+      "text": "uh two outputs are from the preference"
+    },
+    {
+      "start": 4464.6,
+      "duration": 0.0,
+      "text": "uh two outputs are from the preference from<01:14:24.760><c> the</c><01:14:25.080><c> perspective</c><01:14:25.440><c> of</c><01:14:25.560><c> the</c><01:14:25.679><c> human</c><01:14:26.679><c> um</c><01:14:27.159><c> so</c>"
+    },
+    {
+      "start": 4467.669,
+      "duration": 0.0,
+      "text": "from the perspective of the human um so"
+    },
+    {
+      "start": 4467.679,
+      "duration": 0.0,
+      "text": "from the perspective of the human um so this<01:14:27.800><c> is</c><01:14:27.880><c> a</c><01:14:27.960><c> little</c><01:14:28.120><c> bit</c><01:14:28.280><c> meta</c><01:14:28.920><c> but</c><01:14:29.040><c> what</c><01:14:29.159><c> you</c>"
+    },
+    {
+      "start": 4469.31,
+      "duration": 0.0,
+      "text": "this is a little bit meta but what you"
+    },
+    {
+      "start": 4469.32,
+      "duration": 0.0,
+      "text": "this is a little bit meta but what you basically<01:14:29.679><c> do</c><01:14:29.920><c> is</c><01:14:30.000><c> that</c><01:14:30.159><c> you</c><01:14:30.239><c> train</c><01:14:31.080><c> uh</c><01:14:31.320><c> you</c>"
+    },
+    {
+      "start": 4471.47,
+      "duration": 0.0,
+      "text": "basically do is that you train uh you"
+    },
+    {
+      "start": 4471.48,
+      "duration": 0.0,
+      "text": "basically do is that you train uh you take<01:14:31.840><c> um</c><01:14:32.840><c> a</c><01:14:32.960><c> reward</c><01:14:33.280><c> model</c><01:14:33.560><c> R</c><01:14:33.880><c> which</c><01:14:34.000><c> is</c><01:14:34.120><c> a</c><01:14:34.639><c> uh</c>"
+    },
+    {
+      "start": 4474.87,
+      "duration": 0.0,
+      "text": "take um a reward model R which is a uh"
+    },
+    {
+      "start": 4474.88,
+      "duration": 0.0,
+      "text": "take um a reward model R which is a uh just<01:14:35.040><c> a</c><01:14:35.280><c> large</c><01:14:36.080><c> also</c><01:14:36.320><c> a</c><01:14:36.520><c> large</c><01:14:37.159><c> um</c><01:14:37.880><c> a</c><01:14:38.000><c> large</c>"
+    },
+    {
+      "start": 4478.43,
+      "duration": 0.0,
+      "text": "just a large also a large um a large"
+    },
+    {
+      "start": 4478.44,
+      "duration": 0.0,
+      "text": "just a large also a large um a large classifier<01:14:39.440><c> and</c><01:14:39.560><c> you</c><01:14:39.760><c> basically</c><01:14:40.280><c> ask</c><01:14:40.639><c> this</c>"
+    },
+    {
+      "start": 4480.79,
+      "duration": 0.0,
+      "text": "classifier and you basically ask this"
+    },
+    {
+      "start": 4480.8,
+      "duration": 0.0,
+      "text": "classifier and you basically ask this reward<01:14:41.159><c> model</c><01:14:41.480><c> you</c><01:14:41.600><c> give</c><01:14:41.760><c> it</c><01:14:41.960><c> the</c><01:14:42.159><c> input</c><01:14:42.679><c> and</c>"
+    },
+    {
+      "start": 4482.87,
+      "duration": 0.0,
+      "text": "reward model you give it the input and"
+    },
+    {
+      "start": 4482.88,
+      "duration": 0.0,
+      "text": "reward model you give it the input and the<01:14:43.080><c> actual</c><01:14:43.400><c> output</c><01:14:43.800><c> that</c><01:14:43.920><c> you</c><01:14:44.040><c> have</c><01:14:44.280><c> one</c><01:14:44.400><c> of</c>"
+    },
+    {
+      "start": 4484.51,
+      "duration": 0.0,
+      "text": "the actual output that you have one of"
+    },
+    {
+      "start": 4484.52,
+      "duration": 0.0,
+      "text": "the actual output that you have one of the<01:14:44.639><c> two</c><01:14:44.880><c> outputs</c><01:14:45.880><c> uh</c><01:14:46.040><c> and</c><01:14:46.120><c> you</c><01:14:46.360><c> just</c><01:14:47.040><c> um</c>"
+    },
+    {
+      "start": 4487.51,
+      "duration": 0.0,
+      "text": "the two outputs uh and you just um"
+    },
+    {
+      "start": 4487.52,
+      "duration": 0.0,
+      "text": "the two outputs uh and you just um exponentiate<01:14:48.159><c> that</c><01:14:48.320><c> so</c><01:14:48.480><c> that's</c><01:14:48.639><c> the</c><01:14:48.760><c> soft</c><01:14:49.040><c> Max</c>"
+    },
+    {
+      "start": 4489.31,
+      "duration": 0.0,
+      "text": "exponentiate that so that's the soft Max"
+    },
+    {
+      "start": 4489.32,
+      "duration": 0.0,
+      "text": "exponentiate that so that's the soft Max law<01:14:49.679><c> that</c><01:14:49.760><c> you</c><01:14:49.880><c> all</c><01:14:50.040><c> know</c><01:14:50.239><c> about</c><01:14:50.880><c> and</c><01:14:51.080><c> now</c><01:14:51.239><c> you</c>"
+    },
+    {
+      "start": 4491.39,
+      "duration": 0.0,
+      "text": "law that you all know about and now you"
+    },
+    {
+      "start": 4491.4,
+      "duration": 0.0,
+      "text": "law that you all know about and now you divide<01:14:51.840><c> by</c><01:14:52.639><c> um</c><01:14:53.320><c> the</c><01:14:54.239><c> the</c><01:14:54.400><c> exponential</c>"
+    },
+    {
+      "start": 4495.51,
+      "duration": 0.0,
+      "text": "divide by um the the exponential"
+    },
+    {
+      "start": 4495.52,
+      "duration": 0.0,
+      "text": "divide by um the the exponential reward<01:14:56.520><c> uh</c><01:14:56.800><c> on</c><01:14:57.040><c> the</c><01:14:57.360><c> first</c><01:14:58.080><c> example</c><01:14:58.880><c> sorry</c><01:14:59.120><c> on</c>"
+    },
+    {
+      "start": 4499.189,
+      "duration": 0.0,
+      "text": "reward uh on the first example sorry on"
+    },
+    {
+      "start": 4499.199,
+      "duration": 0.0,
+      "text": "reward uh on the first example sorry on the<01:14:59.320><c> first</c><01:14:59.520><c> output</c><01:15:00.080><c> and</c><01:15:00.199><c> this</c><01:15:00.320><c> is</c><01:15:00.400><c> on</c><01:15:00.520><c> the</c>"
+    },
+    {
+      "start": 4500.629,
+      "duration": 0.0,
+      "text": "the first output and this is on the"
+    },
+    {
+      "start": 4500.639,
+      "duration": 0.0,
+      "text": "the first output and this is on the second<01:15:00.880><c> output</c><01:15:01.400><c> and</c><01:15:01.480><c> you</c><01:15:01.639><c> basically</c><01:15:02.000><c> train</c><01:15:02.679><c> so</c>"
+    },
+    {
+      "start": 4502.87,
+      "duration": 0.0,
+      "text": "second output and you basically train so"
+    },
+    {
+      "start": 4502.88,
+      "duration": 0.0,
+      "text": "second output and you basically train so the<01:15:03.080><c> reason</c><01:15:03.320><c> why</c><01:15:03.440><c> you</c><01:15:03.560><c> do</c><01:15:03.719><c> that</c><01:15:03.920><c> is</c><01:15:04.000><c> that</c><01:15:04.120><c> you</c>"
+    },
+    {
+      "start": 4504.31,
+      "duration": 0.0,
+      "text": "the reason why you do that is that you"
+    },
+    {
+      "start": 4504.32,
+      "duration": 0.0,
+      "text": "the reason why you do that is that you train<01:15:04.719><c> your</c><01:15:05.040><c> your</c><01:15:05.199><c> model</c><01:15:05.760><c> you</c><01:15:05.920><c> train</c><01:15:06.199><c> this</c>"
+    },
+    {
+      "start": 4506.35,
+      "duration": 0.0,
+      "text": "train your your model you train this"
+    },
+    {
+      "start": 4506.36,
+      "duration": 0.0,
+      "text": "train your your model you train this reward<01:15:06.719><c> model</c><01:15:07.199><c> to</c><01:15:07.360><c> be</c><01:15:07.520><c> able</c><01:15:07.719><c> to</c><01:15:08.320><c> classify</c><01:15:09.320><c> um</c>"
+    },
+    {
+      "start": 4510.07,
+      "duration": 0.0,
+      "text": "reward model to be able to classify um"
+    },
+    {
+      "start": 4510.08,
+      "duration": 0.0,
+      "text": "reward model to be able to classify um how<01:15:10.679><c> much</c><01:15:11.159><c> better</c><01:15:11.679><c> one</c><01:15:11.960><c> output</c><01:15:12.360><c> is</c><01:15:12.520><c> to</c><01:15:12.719><c> another</c>"
+    },
+    {
+      "start": 4512.99,
+      "duration": 0.0,
+      "text": "how much better one output is to another"
+    },
+    {
+      "start": 4513.0,
+      "duration": 0.0,
+      "text": "how much better one output is to another one<01:15:13.440><c> so</c><01:15:13.679><c> another</c><01:15:14.520><c> uh</c><01:15:14.639><c> slightly</c><01:15:15.000><c> less</c>"
+    },
+    {
+      "start": 4515.149,
+      "duration": 0.0,
+      "text": "one so another uh slightly less"
+    },
+    {
+      "start": 4515.159,
+      "duration": 0.0,
+      "text": "one so another uh slightly less convoluted<01:15:15.719><c> way</c><01:15:15.840><c> of</c><01:15:15.960><c> saying</c><01:15:16.239><c> it</c><01:15:16.520><c> is</c><01:15:16.639><c> that</c><01:15:16.760><c> your</c>"
+    },
+    {
+      "start": 4516.91,
+      "duration": 0.0,
+      "text": "convoluted way of saying it is that your"
+    },
+    {
+      "start": 4516.92,
+      "duration": 0.0,
+      "text": "convoluted way of saying it is that your reward<01:15:17.320><c> model</c><01:15:18.040><c> will</c><01:15:18.639><c> output</c><01:15:19.080><c> some</c><01:15:19.239><c> reward</c>"
+    },
+    {
+      "start": 4519.55,
+      "duration": 0.0,
+      "text": "reward model will output some reward"
+    },
+    {
+      "start": 4519.56,
+      "duration": 0.0,
+      "text": "reward model will output some reward that<01:15:19.679><c> will</c><01:15:19.840><c> be</c><01:15:19.960><c> used</c><01:15:20.239><c> as</c><01:15:20.320><c> the</c><01:15:20.480><c> logits</c><01:15:21.400><c> of</c><01:15:21.560><c> your</c>"
+    },
+    {
+      "start": 4521.75,
+      "duration": 0.0,
+      "text": "that will be used as the logits of your"
+    },
+    {
+      "start": 4521.76,
+      "duration": 0.0,
+      "text": "that will be used as the logits of your soft<01:15:22.120><c> Max</c><01:15:22.719><c> so</c><01:15:22.960><c> now</c><01:15:23.199><c> if</c><01:15:23.320><c> you</c><01:15:23.520><c> have</c><01:15:24.040><c> high</c><01:15:24.520><c> logic</c>"
+    },
+    {
+      "start": 4525.07,
+      "duration": 0.0,
+      "text": "soft Max so now if you have high logic"
+    },
+    {
+      "start": 4525.08,
+      "duration": 0.0,
+      "text": "soft Max so now if you have high logic in<01:15:25.159><c> your</c><01:15:25.280><c> softmax</c><01:15:26.000><c> it</c><01:15:26.080><c> means</c><01:15:26.360><c> that</c><01:15:26.520><c> you</c><01:15:27.239><c> highly</c>"
+    },
+    {
+      "start": 4527.629,
+      "duration": 0.0,
+      "text": "in your softmax it means that you highly"
+    },
+    {
+      "start": 4527.639,
+      "duration": 0.0,
+      "text": "in your softmax it means that you highly likely<01:15:28.520><c> this</c><01:15:29.360><c> um</c><01:15:29.920><c> output</c><01:15:30.480><c> is</c>"
+    },
+    {
+      "start": 4531.669,
+      "duration": 0.0,
+      "text": "likely this um output is"
+    },
+    {
+      "start": 4531.679,
+      "duration": 0.0,
+      "text": "likely this um output is better<01:15:32.679><c> uh</c><01:15:32.800><c> so</c><01:15:32.960><c> that's</c><01:15:33.120><c> what</c><01:15:33.199><c> we</c><01:15:33.320><c> call</c><01:15:33.560><c> Bradley</c>"
+    },
+    {
+      "start": 4533.87,
+      "duration": 0.0,
+      "text": "better uh so that's what we call Bradley"
+    },
+    {
+      "start": 4533.88,
+      "duration": 0.0,
+      "text": "better uh so that's what we call Bradley ter<01:15:34.239><c> model</c><01:15:35.159><c> yes</c><01:15:35.480><c> is</c><01:15:35.600><c> this</c><01:15:35.760><c> reward</c><01:15:36.080><c> model</c><01:15:36.520><c> going</c>"
+    },
+    {
+      "start": 4536.79,
+      "duration": 0.0,
+      "text": "ter model yes is this reward model going"
+    },
+    {
+      "start": 4536.8,
+      "duration": 0.0,
+      "text": "ter model yes is this reward model going over<01:15:37.159><c> the</c><01:15:37.480><c> entire</c><01:15:38.080><c> output</c><01:15:38.520><c> or</c><01:15:38.679><c> is</c><01:15:38.760><c> it</c>"
+    },
+    {
+      "start": 4539.47,
+      "duration": 0.0,
+      "text": "over the entire output or is it"
+    },
+    {
+      "start": 4539.48,
+      "duration": 0.0,
+      "text": "over the entire output or is it going<01:15:40.480><c> um</c><01:15:41.080><c> so</c><01:15:42.080><c> this</c><01:15:42.320><c> takes</c><01:15:42.880><c> the</c>"
+    },
+    {
+      "start": 4543.91,
+      "duration": 0.0,
+      "text": "going um so this takes the"
+    },
+    {
+      "start": 4543.92,
+      "duration": 0.0,
+      "text": "going um so this takes the entire<01:15:44.920><c> uh</c><01:15:45.440><c> yeah</c><01:15:45.560><c> this</c><01:15:45.679><c> takes</c><01:15:45.880><c> the</c><01:15:46.000><c> entire</c>"
+    },
+    {
+      "start": 4546.31,
+      "duration": 0.0,
+      "text": "entire uh yeah this takes the entire"
+    },
+    {
+      "start": 4546.32,
+      "duration": 0.0,
+      "text": "entire uh yeah this takes the entire output<01:15:46.600><c> at</c><01:15:46.719><c> once</c><01:15:46.880><c> so</c><01:15:47.000><c> it</c><01:15:47.080><c> takes</c><01:15:47.320><c> all</c><01:15:47.520><c> the</c><01:15:47.600><c> input</c>"
+    },
+    {
+      "start": 4547.87,
+      "duration": 0.0,
+      "text": "output at once so it takes all the input"
+    },
+    {
+      "start": 4547.88,
+      "duration": 0.0,
+      "text": "output at once so it takes all the input and<01:15:48.000><c> all</c><01:15:48.159><c> the</c><01:15:48.239><c> output</c><01:15:48.679><c> and</c><01:15:48.800><c> it</c><01:15:48.960><c> gives</c><01:15:49.199><c> one</c>"
+    },
+    {
+      "start": 4549.75,
+      "duration": 0.0,
+      "text": "and all the output and it gives one"
+    },
+    {
+      "start": 4549.76,
+      "duration": 0.0,
+      "text": "and all the output and it gives one number"
+    },
+    {
+      "start": 4552.88,
+      "duration": 0.0,
+      "text": "yes<01:15:53.880><c> would</c><01:15:54.080><c> human</c><01:15:54.440><c> be</c><01:15:55.199><c> sorry</c><01:15:55.760><c> with</c><01:15:55.880><c> the</c><01:15:56.080><c> reward</c>"
+    },
+    {
+      "start": 4556.47,
+      "duration": 0.0,
+      "text": "yes would human be sorry with the reward"
+    },
+    {
+      "start": 4556.48,
+      "duration": 0.0,
+      "text": "yes would human be sorry with the reward model<01:15:57.040><c> where</c><01:15:57.159><c> would</c><01:15:57.280><c> a</c><01:15:57.480><c> human</c><01:15:57.840><c> be</c><01:15:58.400><c> like</c><01:15:58.760><c> oh</c><01:15:58.920><c> I</c>"
+    },
+    {
+      "start": 4559.43,
+      "duration": 0.0,
+      "text": "model where would a human be like oh I"
+    },
+    {
+      "start": 4559.44,
+      "duration": 0.0,
+      "text": "model where would a human be like oh I see<01:16:00.440><c> okay</c><01:16:00.600><c> sorry</c><01:16:01.199><c> maybe</c><01:16:01.360><c> I</c><01:16:01.440><c> wasn't</c><01:16:01.719><c> clear</c><01:16:02.520><c> um</c>"
+    },
+    {
+      "start": 4563.03,
+      "duration": 0.0,
+      "text": "see okay sorry maybe I wasn't clear um"
+    },
+    {
+      "start": 4563.04,
+      "duration": 0.0,
+      "text": "see okay sorry maybe I wasn't clear um you<01:16:03.400><c> train</c><01:16:03.800><c> this</c><01:16:03.960><c> reward</c><01:16:04.400><c> model</c><01:16:05.199><c> to</c><01:16:05.639><c> fit</c><01:16:06.440><c> this</c>"
+    },
+    {
+      "start": 4566.709,
+      "duration": 0.0,
+      "text": "you train this reward model to fit this"
+    },
+    {
+      "start": 4566.719,
+      "duration": 0.0,
+      "text": "you train this reward model to fit this green<01:16:07.120><c> and</c><01:16:07.440><c> and</c><01:16:07.920><c> red</c><01:16:08.600><c> preference</c><01:16:09.040><c> from</c><01:16:09.280><c> humans</c>"
+    },
+    {
+      "start": 4569.95,
+      "duration": 0.0,
+      "text": "green and and red preference from humans"
+    },
+    {
+      "start": 4569.96,
+      "duration": 0.0,
+      "text": "green and and red preference from humans so<01:16:10.159><c> basically</c><01:16:10.560><c> you</c><01:16:10.679><c> train</c><01:16:10.960><c> a</c><01:16:11.159><c> classifier</c><01:16:12.040><c> to</c>"
+    },
+    {
+      "start": 4572.229,
+      "duration": 0.0,
+      "text": "so basically you train a classifier to"
+    },
+    {
+      "start": 4572.239,
+      "duration": 0.0,
+      "text": "so basically you train a classifier to say<01:16:12.600><c> whether</c><01:16:13.199><c> the</c><01:16:13.360><c> humans</c><01:16:13.719><c> prefer</c><01:16:14.080><c> red</c><01:16:14.239><c> or</c>"
+    },
+    {
+      "start": 4574.91,
+      "duration": 0.0,
+      "text": "say whether the humans prefer red or"
+    },
+    {
+      "start": 4574.92,
+      "duration": 0.0,
+      "text": "say whether the humans prefer red or green<01:16:15.920><c> uh</c><01:16:16.040><c> but</c><01:16:16.239><c> instead</c><01:16:16.520><c> of</c><01:16:16.639><c> using</c><01:16:16.960><c> the</c><01:16:17.199><c> binary</c>"
+    },
+    {
+      "start": 4577.709,
+      "duration": 0.0,
+      "text": "green uh but instead of using the binary"
+    },
+    {
+      "start": 4577.719,
+      "duration": 0.0,
+      "text": "green uh but instead of using the binary reward<01:16:18.199><c> which</c><01:16:18.320><c> is</c><01:16:18.440><c> what</c><01:16:18.560><c> the</c><01:16:18.679><c> human</c><01:16:19.000><c> would</c>"
+    },
+    {
+      "start": 4579.149,
+      "duration": 0.0,
+      "text": "reward which is what the human would"
+    },
+    {
+      "start": 4579.159,
+      "duration": 0.0,
+      "text": "reward which is what the human would tell<01:16:19.360><c> you</c><01:16:19.880><c> you</c><01:16:20.120><c> basically</c><01:16:20.480><c> use</c><01:16:20.679><c> the</c><01:16:20.880><c> logits</c><01:16:21.880><c> of</c>"
+    },
+    {
+      "start": 4582.03,
+      "duration": 0.0,
+      "text": "tell you you basically use the logits of"
+    },
+    {
+      "start": 4582.04,
+      "duration": 0.0,
+      "text": "tell you you basically use the logits of the<01:16:22.199><c> soft</c><01:16:22.560><c> Max</c><01:16:23.320><c> and</c><01:16:23.440><c> the</c><01:16:23.560><c> thing</c><01:16:23.719><c> with</c><01:16:23.840><c> the</c>"
+    },
+    {
+      "start": 4583.95,
+      "duration": 0.0,
+      "text": "the soft Max and the thing with the"
+    },
+    {
+      "start": 4583.96,
+      "duration": 0.0,
+      "text": "the soft Max and the thing with the logits<01:16:24.639><c> is</c><01:16:24.760><c> that</c><01:16:24.920><c> that</c><01:16:25.040><c> logits</c><01:16:25.520><c> are</c>"
+    },
+    {
+      "start": 4585.75,
+      "duration": 0.0,
+      "text": "logits is that that logits are"
+    },
+    {
+      "start": 4585.76,
+      "duration": 0.0,
+      "text": "logits is that that logits are continuous<01:16:26.719><c> so</c><01:16:26.960><c> now</c><01:16:27.120><c> you</c><01:16:27.239><c> know</c><01:16:27.440><c> that</c><01:16:27.600><c> if</c><01:16:27.679><c> your</c>"
+    },
+    {
+      "start": 4587.83,
+      "duration": 0.0,
+      "text": "continuous so now you know that if your"
+    },
+    {
+      "start": 4587.84,
+      "duration": 0.0,
+      "text": "continuous so now you know that if your reward<01:16:28.199><c> model</c><01:16:28.600><c> said</c><01:16:29.159><c> it</c><01:16:29.360><c> has</c><01:16:29.600><c> high</c><01:16:29.760><c> logits</c>"
+    },
+    {
+      "start": 4590.709,
+      "duration": 0.0,
+      "text": "reward model said it has high logits"
+    },
+    {
+      "start": 4590.719,
+      "duration": 0.0,
+      "text": "reward model said it has high logits then<01:16:30.880><c> in</c><01:16:31.040><c> some</c><01:16:31.280><c> ways</c><01:16:31.560><c> the</c><01:16:31.760><c> human</c><01:16:32.480><c> highly</c>"
+    },
+    {
+      "start": 4592.87,
+      "duration": 0.0,
+      "text": "then in some ways the human highly"
+    },
+    {
+      "start": 4592.88,
+      "duration": 0.0,
+      "text": "then in some ways the human highly prefer<01:16:33.280><c> this</c><01:16:33.480><c> answer</c><01:16:33.920><c> to</c><01:16:34.120><c> some</c><01:16:34.360><c> other</c>"
+    },
+    {
+      "start": 4596.189,
+      "duration": 0.0,
+      "text": "prefer this answer to some other"
+    },
+    {
+      "start": 4596.199,
+      "duration": 0.0,
+      "text": "prefer this answer to some other answer<01:16:37.199><c> great</c><01:16:38.040><c> um</c><01:16:38.960><c> so</c><01:16:39.159><c> as</c><01:16:39.239><c> I</c><01:16:39.400><c> just</c><01:16:39.560><c> said</c>"
+    },
+    {
+      "start": 4599.91,
+      "duration": 0.0,
+      "text": "answer great um so as I just said"
+    },
+    {
+      "start": 4599.92,
+      "duration": 0.0,
+      "text": "answer great um so as I just said continuous<01:16:40.440><c> information</c><01:16:40.880><c> so</c><01:16:41.040><c> it's</c><01:16:41.199><c> better</c><01:16:41.440><c> so</c>"
+    },
+    {
+      "start": 4601.59,
+      "duration": 0.0,
+      "text": "continuous information so it's better so"
+    },
+    {
+      "start": 4601.6,
+      "duration": 0.0,
+      "text": "continuous information so it's better so that's<01:16:41.760><c> what</c><01:16:41.880><c> people</c><01:16:42.600><c> uh</c><01:16:42.719><c> use</c><01:16:42.960><c> in</c><01:16:43.159><c> practice</c><01:16:43.600><c> or</c>"
+    },
+    {
+      "start": 4603.709,
+      "duration": 0.0,
+      "text": "that's what people uh use in practice or"
+    },
+    {
+      "start": 4603.719,
+      "duration": 0.0,
+      "text": "that's what people uh use in practice or at<01:16:43.840><c> least</c><01:16:44.159><c> used</c><01:16:44.400><c> to</c><01:16:44.520><c> use</c><01:16:44.719><c> in</c><01:16:44.880><c> practice</c><01:16:45.560><c> I'll</c>"
+    },
+    {
+      "start": 4605.709,
+      "duration": 0.0,
+      "text": "at least used to use in practice I'll"
+    },
+    {
+      "start": 4605.719,
+      "duration": 0.0,
+      "text": "at least used to use in practice I'll tell<01:16:45.920><c> you</c><01:16:46.120><c> about</c><01:16:46.560><c> uh</c><01:16:46.719><c> the</c><01:16:46.840><c> other</c><01:16:47.040><c> algorithm</c>"
+    },
+    {
+      "start": 4607.55,
+      "duration": 0.0,
+      "text": "tell you about uh the other algorithm"
+    },
+    {
+      "start": 4607.56,
+      "duration": 0.0,
+      "text": "tell you about uh the other algorithm later<01:16:48.320><c> uh</c><01:16:48.400><c> so</c><01:16:48.560><c> what</c><01:16:48.639><c> you</c><01:16:48.719><c> do</c><01:16:48.960><c> at</c><01:16:49.040><c> the</c><01:16:49.159><c> end</c><01:16:49.600><c> is</c>"
+    },
+    {
+      "start": 4609.709,
+      "duration": 0.0,
+      "text": "later uh so what you do at the end is"
+    },
+    {
+      "start": 4609.719,
+      "duration": 0.0,
+      "text": "later uh so what you do at the end is that<01:16:49.880><c> you</c><01:16:50.080><c> basically</c><01:16:50.560><c> try</c><01:16:50.840><c> to</c><01:16:51.600><c> just</c><01:16:51.760><c> use</c>"
+    },
+    {
+      "start": 4611.95,
+      "duration": 0.0,
+      "text": "that you basically try to just use"
+    },
+    {
+      "start": 4611.96,
+      "duration": 0.0,
+      "text": "that you basically try to just use reinforcement<01:16:52.480><c> learning</c><01:16:52.800><c> that</c><01:16:52.880><c> you</c><01:16:53.000><c> know</c>"
+    },
+    {
+      "start": 4613.189,
+      "duration": 0.0,
+      "text": "reinforcement learning that you know"
+    },
+    {
+      "start": 4613.199,
+      "duration": 0.0,
+      "text": "reinforcement learning that you know about<01:16:53.639><c> now</c><01:16:53.800><c> we</c><01:16:53.920><c> know</c><01:16:54.480><c> we</c><01:16:54.639><c> have</c><01:16:55.120><c> reward</c><01:16:55.760><c> what</c>"
+    },
+    {
+      "start": 4615.87,
+      "duration": 0.0,
+      "text": "about now we know we have reward what"
+    },
+    {
+      "start": 4615.88,
+      "duration": 0.0,
+      "text": "about now we know we have reward what you<01:16:56.080><c> sample</c><01:16:56.480><c> through</c><01:16:57.080><c> is</c><01:16:57.280><c> the</c><01:16:57.600><c> generation</c>"
+    },
+    {
+      "start": 4618.03,
+      "duration": 0.0,
+      "text": "you sample through is the generation"
+    },
+    {
+      "start": 4618.04,
+      "duration": 0.0,
+      "text": "you sample through is the generation from<01:16:58.239><c> your</c><01:16:58.360><c> large</c><01:16:58.600><c> language</c><01:16:58.920><c> model</c><01:16:59.920><c> um</c><01:17:00.239><c> and</c>"
+    },
+    {
+      "start": 4620.39,
+      "duration": 0.0,
+      "text": "from your large language model um and"
+    },
+    {
+      "start": 4620.4,
+      "duration": 0.0,
+      "text": "from your large language model um and then<01:17:00.560><c> you</c><01:17:00.679><c> just</c><01:17:00.800><c> use</c><01:17:01.040><c> some</c><01:17:01.199><c> regularization</c>"
+    },
+    {
+      "start": 4621.83,
+      "duration": 0.0,
+      "text": "then you just use some regularization"
+    },
+    {
+      "start": 4621.84,
+      "duration": 0.0,
+      "text": "then you just use some regularization term<01:17:02.120><c> so</c><01:17:02.239><c> the</c><01:17:02.360><c> reason</c><01:17:02.600><c> why</c><01:17:02.719><c> you</c><01:17:03.000><c> do</c><01:17:03.159><c> this</c>"
+    },
+    {
+      "start": 4623.31,
+      "duration": 0.0,
+      "text": "term so the reason why you do this"
+    },
+    {
+      "start": 4623.32,
+      "duration": 0.0,
+      "text": "term so the reason why you do this regularization<01:17:03.960><c> term</c><01:17:04.360><c> is</c><01:17:04.560><c> for</c><01:17:04.920><c> avoiding</c><01:17:05.280><c> what</c>"
+    },
+    {
+      "start": 4625.39,
+      "duration": 0.0,
+      "text": "regularization term is for avoiding what"
+    },
+    {
+      "start": 4625.4,
+      "duration": 0.0,
+      "text": "regularization term is for avoiding what we<01:17:05.520><c> call</c><01:17:05.679><c> over</c><01:17:05.960><c> optimization</c><01:17:06.719><c> so</c><01:17:06.920><c> this</c><01:17:07.120><c> reward</c>"
+    },
+    {
+      "start": 4627.47,
+      "duration": 0.0,
+      "text": "we call over optimization so this reward"
+    },
+    {
+      "start": 4627.48,
+      "duration": 0.0,
+      "text": "we call over optimization so this reward model<01:17:07.800><c> might</c><01:17:07.960><c> not</c><01:17:08.159><c> be</c><01:17:08.360><c> really</c><01:17:08.639><c> represent</c><01:17:09.159><c> like</c>"
+    },
+    {
+      "start": 4629.51,
+      "duration": 0.0,
+      "text": "model might not be really represent like"
+    },
+    {
+      "start": 4629.52,
+      "duration": 0.0,
+      "text": "model might not be really represent like might<01:17:09.679><c> not</c><01:17:10.000><c> perfectly</c><01:17:10.400><c> model</c><01:17:11.040><c> human</c>"
+    },
+    {
+      "start": 4631.35,
+      "duration": 0.0,
+      "text": "might not perfectly model human"
+    },
+    {
+      "start": 4631.36,
+      "duration": 0.0,
+      "text": "might not perfectly model human preferences<01:17:12.199><c> so</c><01:17:12.320><c> you</c><01:17:12.440><c> don't</c><01:17:12.639><c> want</c><01:17:12.719><c> to</c>"
+    },
+    {
+      "start": 4632.95,
+      "duration": 0.0,
+      "text": "preferences so you don't want to"
+    },
+    {
+      "start": 4632.96,
+      "duration": 0.0,
+      "text": "preferences so you don't want to maximize<01:17:13.560><c> this</c><01:17:13.679><c> thing</c><01:17:14.159><c> to</c><01:17:14.600><c> essentially</c>"
+    },
+    {
+      "start": 4635.27,
+      "duration": 0.0,
+      "text": "maximize this thing to essentially"
+    },
+    {
+      "start": 4635.28,
+      "duration": 0.0,
+      "text": "maximize this thing to essentially Infinity<01:17:16.360><c> um</c><01:17:17.360><c> and</c><01:17:17.520><c> you</c><01:17:17.679><c> do</c><01:17:18.080><c> it</c><01:17:18.280><c> using</c><01:17:18.800><c> uh</c><01:17:19.000><c> po</c>"
+    },
+    {
+      "start": 4639.83,
+      "duration": 0.0,
+      "text": "Infinity um and you do it using uh po"
+    },
+    {
+      "start": 4639.84,
+      "duration": 0.0,
+      "text": "Infinity um and you do it using uh po which<01:17:19.960><c> is</c><01:17:20.159><c> a</c><01:17:21.159><c> common</c><01:17:21.960><c> uh</c><01:17:22.239><c> reinforcement</c>"
+    },
+    {
+      "start": 4642.709,
+      "duration": 0.0,
+      "text": "which is a common uh reinforcement"
+    },
+    {
+      "start": 4642.719,
+      "duration": 0.0,
+      "text": "which is a common uh reinforcement learning<01:17:23.239><c> algorithm</c><01:17:24.239><c> um</c><01:17:24.560><c> one</c><01:17:24.960><c> thing</c><01:17:25.080><c> to</c><01:17:25.239><c> note</c>"
+    },
+    {
+      "start": 4645.51,
+      "duration": 0.0,
+      "text": "learning algorithm um one thing to note"
+    },
+    {
+      "start": 4645.52,
+      "duration": 0.0,
+      "text": "learning algorithm um one thing to note here<01:17:25.800><c> because</c><01:17:25.920><c> it</c><01:17:26.000><c> will</c><01:17:26.159><c> be</c><01:17:26.280><c> important</c><01:17:26.560><c> for</c>"
+    },
+    {
+      "start": 4646.75,
+      "duration": 0.0,
+      "text": "here because it will be important for"
+    },
+    {
+      "start": 4646.76,
+      "duration": 0.0,
+      "text": "here because it will be important for later<01:17:27.560><c> is</c><01:17:27.760><c> that</c><01:17:27.960><c> when</c><01:17:28.159><c> we</c><01:17:28.679><c> use</c><01:17:29.080><c> maximum</c>"
+    },
+    {
+      "start": 4649.83,
+      "duration": 0.0,
+      "text": "later is that when we use maximum"
+    },
+    {
+      "start": 4649.84,
+      "duration": 0.0,
+      "text": "later is that when we use maximum likelihood"
+    },
+    {
+      "start": 4651.79,
+      "duration": 0.0,
+      "text": "likelihood"
+    },
+    {
+      "start": 4651.8,
+      "duration": 0.0,
+      "text": "likelihood um<01:17:32.800><c> sorry</c><01:17:33.600><c> now</c><01:17:33.960><c> the</c><01:17:34.159><c> large</c><01:17:34.440><c> language</c><01:17:34.719><c> models</c>"
+    },
+    {
+      "start": 4654.99,
+      "duration": 0.0,
+      "text": "um sorry now the large language models"
+    },
+    {
+      "start": 4655.0,
+      "duration": 0.0,
+      "text": "um sorry now the large language models are<01:17:35.159><c> actually</c><01:17:35.360><c> a</c><01:17:35.639><c> policy</c><01:17:36.639><c> for</c><01:17:36.840><c> your</c>"
+    },
+    {
+      "start": 4656.99,
+      "duration": 0.0,
+      "text": "are actually a policy for your"
+    },
+    {
+      "start": 4657.0,
+      "duration": 0.0,
+      "text": "are actually a policy for your reinforcement<01:17:37.600><c> learning</c><01:17:38.239><c> it's</c><01:17:38.520><c> not</c>"
+    },
+    {
+      "start": 4658.99,
+      "duration": 0.0,
+      "text": "reinforcement learning it's not"
+    },
+    {
+      "start": 4659.0,
+      "duration": 0.0,
+      "text": "reinforcement learning it's not maximizing<01:17:39.639><c> maximum</c><01:17:40.080><c> likelihood</c><01:17:40.520><c> anymore</c>"
+    },
+    {
+      "start": 4661.189,
+      "duration": 0.0,
+      "text": "maximizing maximum likelihood anymore"
+    },
+    {
+      "start": 4661.199,
+      "duration": 0.0,
+      "text": "maximizing maximum likelihood anymore which<01:17:41.360><c> means</c><01:17:41.600><c> that</c><01:17:41.760><c> you're</c><01:17:41.920><c> not</c><01:17:42.159><c> modeling</c><01:17:42.600><c> any</c>"
+    },
+    {
+      "start": 4662.83,
+      "duration": 0.0,
+      "text": "which means that you're not modeling any"
+    },
+    {
+      "start": 4662.84,
+      "duration": 0.0,
+      "text": "which means that you're not modeling any distribution<01:17:43.400><c> anymore</c><01:17:44.199><c> and</c><01:17:44.320><c> the</c><01:17:44.440><c> reason</c><01:17:44.719><c> why</c>"
+    },
+    {
+      "start": 4664.87,
+      "duration": 0.0,
+      "text": "distribution anymore and the reason why"
+    },
+    {
+      "start": 4664.88,
+      "duration": 0.0,
+      "text": "distribution anymore and the reason why this<01:17:44.960><c> is</c><01:17:45.120><c> important</c><01:17:45.440><c> is</c><01:17:45.560><c> that</c><01:17:45.840><c> models</c><01:17:46.239><c> that</c>"
+    },
+    {
+      "start": 4666.39,
+      "duration": 0.0,
+      "text": "this is important is that models that"
+    },
+    {
+      "start": 4666.4,
+      "duration": 0.0,
+      "text": "this is important is that models that went<01:17:46.679><c> through</c><01:17:47.280><c> this</c><01:17:47.480><c> type</c><01:17:47.639><c> of</c><01:17:47.920><c> Po</c><01:17:48.920><c> actually</c>"
+    },
+    {
+      "start": 4669.189,
+      "duration": 0.0,
+      "text": "went through this type of Po actually"
+    },
+    {
+      "start": 4669.199,
+      "duration": 0.0,
+      "text": "went through this type of Po actually don't<01:17:49.560><c> give</c><01:17:49.760><c> you</c><01:17:50.400><c> likelihoods</c><01:17:50.960><c> of</c><01:17:51.199><c> text</c><01:17:51.719><c> that</c>"
+    },
+    {
+      "start": 4671.83,
+      "duration": 0.0,
+      "text": "don't give you likelihoods of text that"
+    },
+    {
+      "start": 4671.84,
+      "duration": 0.0,
+      "text": "don't give you likelihoods of text that are<01:17:52.000><c> meaningful</c><01:17:52.760><c> cuz</c><01:17:53.040><c> what</c><01:17:53.159><c> you</c><01:17:53.400><c> optimize</c>"
+    },
+    {
+      "start": 4673.91,
+      "duration": 0.0,
+      "text": "are meaningful cuz what you optimize"
+    },
+    {
+      "start": 4673.92,
+      "duration": 0.0,
+      "text": "are meaningful cuz what you optimize them<01:17:54.239><c> to</c><01:17:54.400><c> do</c><01:17:54.560><c> is</c><01:17:54.679><c> B</c><01:17:54.840><c> basically</c><01:17:55.199><c> just</c><01:17:55.440><c> optimized</c>"
+    },
+    {
+      "start": 4675.95,
+      "duration": 0.0,
+      "text": "them to do is B basically just optimized"
+    },
+    {
+      "start": 4675.96,
+      "duration": 0.0,
+      "text": "them to do is B basically just optimized for<01:17:56.239><c> generating</c><01:17:56.760><c> the</c><01:17:56.920><c> most</c><01:17:57.199><c> likely</c><01:17:57.560><c> thing</c><01:17:58.480><c> not</c>"
+    },
+    {
+      "start": 4678.79,
+      "duration": 0.0,
+      "text": "for generating the most likely thing not"
+    },
+    {
+      "start": 4678.8,
+      "duration": 0.0,
+      "text": "for generating the most likely thing not optimize<01:17:59.360><c> for</c><01:17:59.560><c> modeling</c><01:18:00.080><c> like</c><01:18:00.560><c> all</c><01:18:00.719><c> the</c>"
+    },
+    {
+      "start": 4680.91,
+      "duration": 0.0,
+      "text": "optimize for modeling like all the"
+    },
+    {
+      "start": 4680.92,
+      "duration": 0.0,
+      "text": "optimize for modeling like all the answers<01:18:01.239><c> that</c><01:18:01.400><c> humans</c><01:18:01.760><c> might</c><01:18:01.960><c> say</c><01:18:02.440><c> another</c>"
+    },
+    {
+      "start": 4682.709,
+      "duration": 0.0,
+      "text": "answers that humans might say another"
+    },
+    {
+      "start": 4682.719,
+      "duration": 0.0,
+      "text": "answers that humans might say another way<01:18:02.920><c> of</c><01:18:03.040><c> saying</c><01:18:03.320><c> that</c><01:18:03.679><c> is</c><01:18:03.840><c> that</c><01:18:04.000><c> there's</c>"
+    },
+    {
+      "start": 4684.31,
+      "duration": 0.0,
+      "text": "way of saying that is that there's"
+    },
+    {
+      "start": 4684.32,
+      "duration": 0.0,
+      "text": "way of saying that is that there's nothing<01:18:04.679><c> that</c><01:18:04.880><c> incentivizes</c><01:18:05.760><c> here</c><01:18:06.080><c> the</c><01:18:06.280><c> model</c>"
+    },
+    {
+      "start": 4686.91,
+      "duration": 0.0,
+      "text": "nothing that incentivizes here the model"
+    },
+    {
+      "start": 4686.92,
+      "duration": 0.0,
+      "text": "nothing that incentivizes here the model to<01:18:07.159><c> not</c><01:18:07.520><c> give</c><01:18:08.120><c> a</c><01:18:08.480><c> like</c><01:18:08.719><c> a</c><01:18:09.239><c> um</c><01:18:09.719><c> a</c><01:18:09.880><c> single</c>"
+    },
+    {
+      "start": 4690.31,
+      "duration": 0.0,
+      "text": "to not give a like a um a single"
+    },
+    {
+      "start": 4690.32,
+      "duration": 0.0,
+      "text": "to not give a like a um a single possible<01:18:10.719><c> generation</c><01:18:11.600><c> nothing</c><01:18:12.000><c> here</c><01:18:12.719><c> says</c>"
+    },
+    {
+      "start": 4693.11,
+      "duration": 0.0,
+      "text": "possible generation nothing here says"
+    },
+    {
+      "start": 4693.12,
+      "duration": 0.0,
+      "text": "possible generation nothing here says it's<01:18:13.400><c> good</c><01:18:13.920><c> if</c><01:18:14.040><c> you</c><01:18:14.239><c> have</c><01:18:14.520><c> some</c><01:18:14.760><c> distribution</c>"
+    },
+    {
+      "start": 4695.27,
+      "duration": 0.0,
+      "text": "it's good if you have some distribution"
+    },
+    {
+      "start": 4695.28,
+      "duration": 0.0,
+      "text": "it's good if you have some distribution with<01:18:15.440><c> some</c>"
+    },
+    {
+      "start": 4696.189,
+      "duration": 0.0,
+      "text": "with some"
+    },
+    {
+      "start": 4696.199,
+      "duration": 0.0,
+      "text": "with some entropy<01:18:17.199><c> um</c><01:18:17.880><c> okay</c><01:18:18.080><c> if</c><01:18:18.120><c> you</c><01:18:18.239><c> haven't</c><01:18:18.400><c> followed</c>"
+    },
+    {
+      "start": 4698.669,
+      "duration": 0.0,
+      "text": "entropy um okay if you haven't followed"
+    },
+    {
+      "start": 4698.679,
+      "duration": 0.0,
+      "text": "entropy um okay if you haven't followed it's<01:18:18.840><c> not</c><01:18:19.040><c> that</c><01:18:19.199><c> important</c><01:18:19.639><c> but</c><01:18:20.040><c> just</c><01:18:20.360><c> good</c><01:18:20.480><c> to</c>"
+    },
+    {
+      "start": 4701.189,
+      "duration": 0.0,
+      "text": "it's not that important but just good to"
+    },
+    {
+      "start": 4701.199,
+      "duration": 0.0,
+      "text": "it's not that important but just good to knowe<01:18:22.440><c> great</c><01:18:23.440><c> so</c><01:18:23.840><c> PO</c><01:18:24.320><c> is</c><01:18:24.480><c> exact</c><01:18:25.080><c> what</c><01:18:25.400><c> chat</c><01:18:25.679><c> GPT</c>"
+    },
+    {
+      "start": 4706.189,
+      "duration": 0.0,
+      "text": "knowe great so PO is exact what chat GPT"
+    },
+    {
+      "start": 4706.199,
+      "duration": 0.0,
+      "text": "knowe great so PO is exact what chat GPT did<01:18:26.560><c> originally</c><01:18:27.320><c> so</c><01:18:27.600><c> here's</c><01:18:27.960><c> the</c><01:18:28.199><c> on</c><01:18:28.400><c> the</c><01:18:28.560><c> blog</c>"
+    },
+    {
+      "start": 4708.83,
+      "duration": 0.0,
+      "text": "did originally so here's the on the blog"
+    },
+    {
+      "start": 4708.84,
+      "duration": 0.0,
+      "text": "did originally so here's the on the blog post<01:18:29.360><c> or</c><01:18:30.159><c> what</c><01:18:30.320><c> they</c><01:18:30.520><c> have</c><01:18:30.920><c> is</c><01:18:31.159><c> step</c><01:18:31.440><c> one</c><01:18:31.880><c> do</c>"
+    },
+    {
+      "start": 4712.149,
+      "duration": 0.0,
+      "text": "post or what they have is step one do"
+    },
+    {
+      "start": 4712.159,
+      "duration": 0.0,
+      "text": "post or what they have is step one do supervise<01:18:32.639><c> fine</c><01:18:32.840><c> training</c><01:18:33.360><c> which</c><01:18:33.520><c> now</c><01:18:33.679><c> you</c>"
+    },
+    {
+      "start": 4713.79,
+      "duration": 0.0,
+      "text": "supervise fine training which now you"
+    },
+    {
+      "start": 4713.8,
+      "duration": 0.0,
+      "text": "supervise fine training which now you all<01:18:33.960><c> know</c><01:18:34.159><c> about</c><01:18:34.800><c> step</c><01:18:35.040><c> two</c><01:18:35.480><c> train</c><01:18:35.760><c> a</c><01:18:35.920><c> reward</c>"
+    },
+    {
+      "start": 4716.27,
+      "duration": 0.0,
+      "text": "all know about step two train a reward"
+    },
+    {
+      "start": 4716.28,
+      "duration": 0.0,
+      "text": "all know about step two train a reward model<01:18:36.600><c> on</c><01:18:36.800><c> human</c><01:18:37.080><c> preferences</c><01:18:38.040><c> step</c><01:18:38.320><c> three</c><01:18:38.840><c> do</c>"
+    },
+    {
+      "start": 4719.189,
+      "duration": 0.0,
+      "text": "model on human preferences step three do"
+    },
+    {
+      "start": 4719.199,
+      "duration": 0.0,
+      "text": "model on human preferences step three do po<01:18:39.960><c> multiple</c><01:18:40.400><c> steps</c><01:18:40.880><c> which</c><01:18:41.040><c> is</c><01:18:41.199><c> where</c><01:18:41.360><c> you</c><01:18:41.480><c> see</c>"
+    },
+    {
+      "start": 4721.75,
+      "duration": 0.0,
+      "text": "po multiple steps which is where you see"
+    },
+    {
+      "start": 4721.76,
+      "duration": 0.0,
+      "text": "po multiple steps which is where you see this<01:18:42.280><c> this</c><01:18:42.440><c> blue</c><01:18:42.719><c> arrow</c><01:18:43.120><c> so</c><01:18:43.280><c> you</c><01:18:43.400><c> continue</c><01:18:43.880><c> you</c>"
+    },
+    {
+      "start": 4723.95,
+      "duration": 0.0,
+      "text": "this this blue arrow so you continue you"
+    },
+    {
+      "start": 4723.96,
+      "duration": 0.0,
+      "text": "this this blue arrow so you continue you train<01:18:44.199><c> the</c><01:18:44.320><c> model</c><01:18:44.560><c> once</c><01:18:44.719><c> with</c><01:18:44.960><c> po</c><01:18:45.520><c> you</c><01:18:45.639><c> collect</c>"
+    },
+    {
+      "start": 4725.91,
+      "duration": 0.0,
+      "text": "train the model once with po you collect"
+    },
+    {
+      "start": 4725.92,
+      "duration": 0.0,
+      "text": "train the model once with po you collect new<01:18:46.120><c> data</c><01:18:46.360><c> you</c><01:18:46.480><c> continue</c><01:18:47.480><c> uh</c><01:18:47.639><c> and</c><01:18:47.800><c> that's</c><01:18:48.040><c> why</c>"
+    },
+    {
+      "start": 4728.31,
+      "duration": 0.0,
+      "text": "new data you continue uh and that's why"
+    },
+    {
+      "start": 4728.32,
+      "duration": 0.0,
+      "text": "new data you continue uh and that's why and<01:18:48.440><c> that's</c><01:18:48.639><c> exactly</c><01:18:48.960><c> what</c><01:18:49.080><c> Chad</c><01:18:49.320><c> GPT</c><01:18:49.719><c> did</c><01:18:50.480><c> uh</c>"
+    },
+    {
+      "start": 4730.629,
+      "duration": 0.0,
+      "text": "and that's exactly what Chad GPT did uh"
+    },
+    {
+      "start": 4730.639,
+      "duration": 0.0,
+      "text": "and that's exactly what Chad GPT did uh that<01:18:50.760><c> was</c><01:18:50.880><c> a</c><01:18:51.120><c> big</c><01:18:51.320><c> breakthrough</c><01:18:52.199><c> between</c><01:18:52.800><c> gpt3</c>"
+    },
+    {
+      "start": 4733.55,
+      "duration": 0.0,
+      "text": "that was a big breakthrough between gpt3"
+    },
+    {
+      "start": 4733.56,
+      "duration": 0.0,
+      "text": "that was a big breakthrough between gpt3 and<01:18:53.679><c> Chad</c><01:18:53.920><c> GPT</c>"
+    },
+    {
+      "start": 4735.35,
+      "duration": 0.0,
+      "text": "and Chad GPT"
+    },
+    {
+      "start": 4735.36,
+      "duration": 0.0,
+      "text": "and Chad GPT one<01:18:55.560><c> thing</c><01:18:55.679><c> to</c><01:18:55.880><c> note</c><01:18:56.440><c> is</c><01:18:56.719><c> that</c><01:18:57.199><c> uh</c><01:18:57.360><c> P</c><01:18:57.800><c> has</c><01:18:57.920><c> many</c>"
+    },
+    {
+      "start": 4738.149,
+      "duration": 0.0,
+      "text": "one thing to note is that uh P has many"
+    },
+    {
+      "start": 4738.159,
+      "duration": 0.0,
+      "text": "one thing to note is that uh P has many challenges<01:18:59.040><c> reinforcement</c><01:18:59.679><c> learning</c><01:18:59.960><c> is</c>"
+    },
+    {
+      "start": 4740.11,
+      "duration": 0.0,
+      "text": "challenges reinforcement learning is"
+    },
+    {
+      "start": 4740.12,
+      "duration": 0.0,
+      "text": "challenges reinforcement learning is something<01:19:00.360><c> that's</c><01:19:00.719><c> super</c><01:19:01.159><c> nice</c>"
+    },
+    {
+      "start": 4741.51,
+      "duration": 0.0,
+      "text": "something that's super nice"
+    },
+    {
+      "start": 4741.52,
+      "duration": 0.0,
+      "text": "something that's super nice theoretically<01:19:02.320><c> in</c><01:19:02.520><c> practice</c><01:19:02.960><c> anyone</c><01:19:03.239><c> who</c>"
+    },
+    {
+      "start": 4743.39,
+      "duration": 0.0,
+      "text": "theoretically in practice anyone who"
+    },
+    {
+      "start": 4743.4,
+      "duration": 0.0,
+      "text": "theoretically in practice anyone who ever<01:19:03.600><c> worked</c><01:19:03.800><c> with</c><01:19:03.920><c> reinforcement</c><01:19:04.440><c> learning</c>"
+    },
+    {
+      "start": 4744.709,
+      "duration": 0.0,
+      "text": "ever worked with reinforcement learning"
+    },
+    {
+      "start": 4744.719,
+      "duration": 0.0,
+      "text": "ever worked with reinforcement learning knows<01:19:05.000><c> it's</c><01:19:05.320><c> such</c><01:19:05.520><c> a</c><01:19:05.679><c> mess</c><01:19:06.400><c> uh</c><01:19:06.520><c> there's</c><01:19:06.719><c> a</c><01:19:06.880><c> lot</c>"
+    },
+    {
+      "start": 4747.03,
+      "duration": 0.0,
+      "text": "knows it's such a mess uh there's a lot"
+    },
+    {
+      "start": 4747.04,
+      "duration": 0.0,
+      "text": "knows it's such a mess uh there's a lot of<01:19:07.199><c> things</c><01:19:07.480><c> like</c><01:19:07.719><c> roll</c><01:19:07.960><c> outs</c><01:19:08.280><c> out</c><01:19:08.480><c> of</c><01:19:08.639><c> Loops</c>"
+    },
+    {
+      "start": 4748.91,
+      "duration": 0.0,
+      "text": "of things like roll outs out of Loops"
+    },
+    {
+      "start": 4748.92,
+      "duration": 0.0,
+      "text": "of things like roll outs out of Loops clipping<01:19:09.600><c> so</c><01:19:09.800><c> many</c><01:19:10.440><c> complications</c><01:19:11.440><c> um</c><01:19:12.120><c> so</c>"
+    },
+    {
+      "start": 4752.27,
+      "duration": 0.0,
+      "text": "clipping so many complications um so"
+    },
+    {
+      "start": 4752.28,
+      "duration": 0.0,
+      "text": "clipping so many complications um so it's<01:19:12.440><c> messy</c><01:19:13.159><c> this</c><01:19:13.280><c> is</c><01:19:13.480><c> the</c><01:19:13.679><c> idealized</c><01:19:14.280><c> PO</c><01:19:14.679><c> used</c>"
+    },
+    {
+      "start": 4754.87,
+      "duration": 0.0,
+      "text": "it's messy this is the idealized PO used"
+    },
+    {
+      "start": 4754.88,
+      "duration": 0.0,
+      "text": "it's messy this is the idealized PO used for<01:19:15.080><c> LM</c><01:19:15.480><c> settings</c><01:19:15.840><c> so</c><01:19:16.040><c> that's</c><01:19:16.239><c> already</c><01:19:16.560><c> much</c>"
+    },
+    {
+      "start": 4756.709,
+      "duration": 0.0,
+      "text": "for LM settings so that's already much"
+    },
+    {
+      "start": 4756.719,
+      "duration": 0.0,
+      "text": "for LM settings so that's already much more<01:19:16.920><c> complicated</c><01:19:17.440><c> than</c><01:19:17.560><c> this</c><01:19:17.800><c> expectation</c>"
+    },
+    {
+      "start": 4758.229,
+      "duration": 0.0,
+      "text": "more complicated than this expectation"
+    },
+    {
+      "start": 4758.239,
+      "duration": 0.0,
+      "text": "more complicated than this expectation we<01:19:18.320><c> saw</c><01:19:18.560><c> before</c><01:19:19.199><c> and</c><01:19:19.320><c> in</c><01:19:19.520><c> practice</c><01:19:19.800><c> it's</c>"
+    },
+    {
+      "start": 4759.95,
+      "duration": 0.0,
+      "text": "we saw before and in practice it's"
+    },
+    {
+      "start": 4759.96,
+      "duration": 0.0,
+      "text": "we saw before and in practice it's actually<01:19:20.199><c> much</c><01:19:20.360><c> more</c><01:19:20.560><c> complicated</c><01:19:21.000><c> so</c><01:19:21.120><c> we</c>"
+    },
+    {
+      "start": 4761.27,
+      "duration": 0.0,
+      "text": "actually much more complicated so we"
+    },
+    {
+      "start": 4761.28,
+      "duration": 0.0,
+      "text": "actually much more complicated so we have<01:19:21.440><c> one</c><01:19:21.639><c> implementation</c><01:19:22.239><c> of</c><01:19:22.360><c> it</c><01:19:22.520><c> that</c><01:19:22.600><c> we</c>"
+    },
+    {
+      "start": 4762.75,
+      "duration": 0.0,
+      "text": "have one implementation of it that we"
+    },
+    {
+      "start": 4762.76,
+      "duration": 0.0,
+      "text": "have one implementation of it that we had<01:19:22.880><c> to</c><01:19:23.000><c> do</c><01:19:23.639><c> and</c><01:19:23.760><c> I'm</c><01:19:24.080><c> not</c><01:19:24.239><c> going</c><01:19:24.320><c> to</c><01:19:24.480><c> go</c>"
+    },
+    {
+      "start": 4764.79,
+      "duration": 0.0,
+      "text": "had to do and I'm not going to go"
+    },
+    {
+      "start": 4764.8,
+      "duration": 0.0,
+      "text": "had to do and I'm not going to go through<01:19:24.920><c> it</c><01:19:25.080><c> but</c><01:19:25.199><c> basically</c><01:19:25.560><c> you</c><01:19:25.679><c> have</c><01:19:25.880><c> like</c>"
+    },
+    {
+      "start": 4766.149,
+      "duration": 0.0,
+      "text": "through it but basically you have like"
+    },
+    {
+      "start": 4766.159,
+      "duration": 0.0,
+      "text": "through it but basically you have like so<01:19:26.360><c> much</c><01:19:26.639><c> stuff</c><01:19:26.880><c> that</c><01:19:27.000><c> you</c><01:19:27.159><c> have</c><01:19:27.320><c> to</c><01:19:27.560><c> think</c>"
+    },
+    {
+      "start": 4767.79,
+      "duration": 0.0,
+      "text": "so much stuff that you have to think"
+    },
+    {
+      "start": 4767.8,
+      "duration": 0.0,
+      "text": "so much stuff that you have to think about<01:19:28.280><c> when</c><01:19:28.440><c> you</c><01:19:28.600><c> implement</c><01:19:29.159><c> that</c><01:19:29.360><c> type</c><01:19:29.639><c> of</c><01:19:30.440><c> of</c>"
+    },
+    {
+      "start": 4770.79,
+      "duration": 0.0,
+      "text": "about when you implement that type of of"
+    },
+    {
+      "start": 4770.8,
+      "duration": 0.0,
+      "text": "about when you implement that type of of uh<01:19:30.960><c> po</c><01:19:31.360><c> algorithm</c><01:19:31.760><c> so</c><01:19:31.880><c> you</c><01:19:31.960><c> have</c><01:19:32.080><c> clipping</c>"
+    },
+    {
+      "start": 4772.47,
+      "duration": 0.0,
+      "text": "uh po algorithm so you have clipping"
+    },
+    {
+      "start": 4772.48,
+      "duration": 0.0,
+      "text": "uh po algorithm so you have clipping everywhere<01:19:33.120><c> you</c><01:19:33.280><c> have</c><01:19:33.560><c> a</c><01:19:33.679><c> lot</c><01:19:33.800><c> of</c>"
+    },
+    {
+      "start": 4774.07,
+      "duration": 0.0,
+      "text": "everywhere you have a lot of"
+    },
+    {
+      "start": 4774.08,
+      "duration": 0.0,
+      "text": "everywhere you have a lot of complexities<01:19:35.080><c> and</c><01:19:35.440><c> things</c><01:19:35.600><c> are</c><01:19:35.719><c> not</c><01:19:35.880><c> well</c>"
+    },
+    {
+      "start": 4776.59,
+      "duration": 0.0,
+      "text": "complexities and things are not well"
+    },
+    {
+      "start": 4776.6,
+      "duration": 0.0,
+      "text": "complexities and things are not well documented<01:19:37.600><c> all</c><01:19:37.840><c> this</c><01:19:38.000><c> to</c><01:19:38.239><c> say</c><01:19:39.120><c> um</c><01:19:39.560><c> that</c><01:19:39.880><c> we're</c>"
+    },
+    {
+      "start": 4780.07,
+      "duration": 0.0,
+      "text": "documented all this to say um that we're"
+    },
+    {
+      "start": 4780.08,
+      "duration": 0.0,
+      "text": "documented all this to say um that we're going<01:19:40.159><c> to</c><01:19:40.480><c> there</c><01:19:40.679><c> was</c><01:19:40.800><c> a</c><01:19:40.960><c> new</c><01:19:41.320><c> method</c><01:19:41.679><c> that</c><01:19:41.800><c> was</c>"
+    },
+    {
+      "start": 4781.95,
+      "duration": 0.0,
+      "text": "going to there was a new method that was"
+    },
+    {
+      "start": 4781.96,
+      "duration": 0.0,
+      "text": "going to there was a new method that was proposed<01:19:42.639><c> uh</c><01:19:42.760><c> also</c><01:19:43.280><c> from</c><01:19:43.440><c> Sanford</c><01:19:44.000><c> one</c><01:19:44.199><c> year</c>"
+    },
+    {
+      "start": 4784.35,
+      "duration": 0.0,
+      "text": "proposed uh also from Sanford one year"
+    },
+    {
+      "start": 4784.36,
+      "duration": 0.0,
+      "text": "proposed uh also from Sanford one year ago<01:19:44.760><c> called</c><01:19:45.000><c> DPO</c><01:19:45.840><c> which</c><01:19:45.960><c> is</c><01:19:46.120><c> essentially</c><01:19:46.440><c> a</c>"
+    },
+    {
+      "start": 4786.59,
+      "duration": 0.0,
+      "text": "ago called DPO which is essentially a"
+    },
+    {
+      "start": 4786.6,
+      "duration": 0.0,
+      "text": "ago called DPO which is essentially a simplification<01:19:47.600><c> of</c><01:19:48.000><c> Po</c><01:19:49.000><c> um</c><01:19:49.880><c> and</c><01:19:50.040><c> the</c><01:19:50.280><c> way</c><01:19:51.280><c> uh</c>"
+    },
+    {
+      "start": 4791.669,
+      "duration": 0.0,
+      "text": "simplification of Po um and the way uh"
+    },
+    {
+      "start": 4791.679,
+      "duration": 0.0,
+      "text": "simplification of Po um and the way uh what<01:19:51.840><c> they</c><01:19:52.000><c> did</c><01:19:52.360><c> or</c><01:19:52.520><c> the</c><01:19:52.639><c> idea</c><01:19:52.920><c> that</c><01:19:53.080><c> they</c><01:19:53.239><c> have</c>"
+    },
+    {
+      "start": 4793.709,
+      "duration": 0.0,
+      "text": "what they did or the idea that they have"
+    },
+    {
+      "start": 4793.719,
+      "duration": 0.0,
+      "text": "what they did or the idea that they have is<01:19:53.880><c> that</c><01:19:54.080><c> instead</c><01:19:54.320><c> of</c><01:19:54.440><c> using</c><01:19:54.840><c> reinforcement</c>"
+    },
+    {
+      "start": 4795.83,
+      "duration": 0.0,
+      "text": "is that instead of using reinforcement"
+    },
+    {
+      "start": 4795.84,
+      "duration": 0.0,
+      "text": "is that instead of using reinforcement learning<01:19:56.280><c> you</c><01:19:56.360><c> can</c><01:19:56.520><c> just</c><01:19:56.719><c> maximize</c><01:19:57.199><c> the</c>"
+    },
+    {
+      "start": 4797.31,
+      "duration": 0.0,
+      "text": "learning you can just maximize the"
+    },
+    {
+      "start": 4797.32,
+      "duration": 0.0,
+      "text": "learning you can just maximize the probability<01:19:57.719><c> of</c><01:19:57.840><c> generating</c><01:19:58.360><c> the</c><01:19:58.560><c> stuff</c><01:19:58.840><c> that</c>"
+    },
+    {
+      "start": 4798.91,
+      "duration": 0.0,
+      "text": "probability of generating the stuff that"
+    },
+    {
+      "start": 4798.92,
+      "duration": 0.0,
+      "text": "probability of generating the stuff that you<01:19:59.120><c> like</c><01:19:59.600><c> and</c><01:19:59.760><c> minimizing</c><01:20:00.239><c> the</c><01:20:00.360><c> probability</c>"
+    },
+    {
+      "start": 4800.669,
+      "duration": 0.0,
+      "text": "you like and minimizing the probability"
+    },
+    {
+      "start": 4800.679,
+      "duration": 0.0,
+      "text": "you like and minimizing the probability of<01:20:00.760><c> the</c><01:20:00.880><c> stuff</c><01:20:01.040><c> that</c><01:20:01.120><c> you</c><01:20:01.239><c> don't</c><01:20:01.520><c> like</c><01:20:02.320><c> uh</c><01:20:02.480><c> so</c>"
+    },
+    {
+      "start": 4802.91,
+      "duration": 0.0,
+      "text": "of the stuff that you don't like uh so"
+    },
+    {
+      "start": 4802.92,
+      "duration": 0.0,
+      "text": "of the stuff that you don't like uh so if<01:20:03.000><c> you</c><01:20:03.120><c> think</c><01:20:03.280><c> about</c><01:20:03.440><c> the</c><01:20:03.520><c> human</c><01:20:03.760><c> preference</c>"
+    },
+    {
+      "start": 4804.229,
+      "duration": 0.0,
+      "text": "if you think about the human preference"
+    },
+    {
+      "start": 4804.239,
+      "duration": 0.0,
+      "text": "if you think about the human preference the<01:20:04.360><c> red</c><01:20:04.520><c> and</c><01:20:04.719><c> green</c><01:20:05.440><c> maximize</c><01:20:06.440><c> uh</c><01:20:06.560><c> green</c>"
+    },
+    {
+      "start": 4806.99,
+      "duration": 0.0,
+      "text": "the red and green maximize uh green"
+    },
+    {
+      "start": 4807.0,
+      "duration": 0.0,
+      "text": "the red and green maximize uh green minimize<01:20:07.600><c> red</c><01:20:08.600><c> um</c><01:20:09.000><c> so</c><01:20:09.199><c> the</c><01:20:09.360><c> loss</c><01:20:09.920><c> is</c><01:20:10.159><c> actually</c>"
+    },
+    {
+      "start": 4810.709,
+      "duration": 0.0,
+      "text": "minimize red um so the loss is actually"
+    },
+    {
+      "start": 4810.719,
+      "duration": 0.0,
+      "text": "minimize red um so the loss is actually this<01:20:10.880><c> one</c><01:20:11.639><c> uh</c><01:20:11.800><c> where</c><01:20:12.000><c> what</c><01:20:12.080><c> you</c><01:20:12.199><c> see</c><01:20:12.560><c> this</c><01:20:12.679><c> is</c>"
+    },
+    {
+      "start": 4812.91,
+      "duration": 0.0,
+      "text": "this one uh where what you see this is"
+    },
+    {
+      "start": 4812.92,
+      "duration": 0.0,
+      "text": "this one uh where what you see this is simply<01:20:13.679><c> um</c><01:20:14.440><c> some</c><01:20:14.920><c> log</c><01:20:15.320><c> of</c><01:20:15.560><c> the</c><01:20:16.040><c> model</c><01:20:16.719><c> so</c><01:20:16.880><c> this</c>"
+    },
+    {
+      "start": 4816.99,
+      "duration": 0.0,
+      "text": "simply um some log of the model so this"
+    },
+    {
+      "start": 4817.0,
+      "duration": 0.0,
+      "text": "simply um some log of the model so this is<01:20:17.120><c> the</c><01:20:17.239><c> likelihood</c><01:20:17.639><c> of</c><01:20:17.760><c> a</c><01:20:17.880><c> model</c><01:20:18.159><c> generating</c>"
+    },
+    {
+      "start": 4818.669,
+      "duration": 0.0,
+      "text": "is the likelihood of a model generating"
+    },
+    {
+      "start": 4818.679,
+      "duration": 0.0,
+      "text": "is the likelihood of a model generating the<01:20:18.840><c> things</c><01:20:19.120><c> that</c><01:20:19.239><c> the</c><01:20:19.400><c> human</c><01:20:19.719><c> preferred</c>"
+    },
+    {
+      "start": 4820.43,
+      "duration": 0.0,
+      "text": "the things that the human preferred"
+    },
+    {
+      "start": 4820.44,
+      "duration": 0.0,
+      "text": "the things that the human preferred given<01:20:20.840><c> the</c><01:20:21.199><c> the</c><01:20:21.480><c> inputs</c><01:20:22.679><c> um</c><01:20:23.679><c> and</c><01:20:24.120><c> what</c><01:20:24.239><c> you</c><01:20:24.400><c> try</c>"
+    },
+    {
+      "start": 4824.83,
+      "duration": 0.0,
+      "text": "given the the inputs um and what you try"
+    },
+    {
+      "start": 4824.84,
+      "duration": 0.0,
+      "text": "given the the inputs um and what you try to<01:20:24.920><c> do</c><01:20:25.040><c> is</c><01:20:25.199><c> basically</c>"
+    },
+    {
+      "start": 4826.229,
+      "duration": 0.0,
+      "text": "to do is basically"
+    },
+    {
+      "start": 4826.239,
+      "duration": 0.0,
+      "text": "to do is basically maximize<01:20:27.239><c> uh</c><01:20:27.679><c> the</c><01:20:28.600><c> likelihood</c><01:20:29.320><c> of</c><01:20:29.480><c> generating</c>"
+    },
+    {
+      "start": 4829.83,
+      "duration": 0.0,
+      "text": "maximize uh the likelihood of generating"
+    },
+    {
+      "start": 4829.84,
+      "duration": 0.0,
+      "text": "maximize uh the likelihood of generating the<01:20:29.920><c> things</c><01:20:30.120><c> that</c><01:20:30.199><c> you</c><01:20:30.360><c> like</c><01:20:30.600><c> minimize</c><01:20:31.040><c> the</c>"
+    },
+    {
+      "start": 4831.149,
+      "duration": 0.0,
+      "text": "the things that you like minimize the"
+    },
+    {
+      "start": 4831.159,
+      "duration": 0.0,
+      "text": "the things that you like minimize the likelihood<01:20:31.520><c> of</c><01:20:31.639><c> the</c><01:20:31.719><c> things</c><01:20:31.920><c> that</c><01:20:32.040><c> you</c><01:20:32.159><c> don't</c>"
+    },
+    {
+      "start": 4832.39,
+      "duration": 0.0,
+      "text": "likelihood of the things that you don't"
+    },
+    {
+      "start": 4832.4,
+      "duration": 0.0,
+      "text": "likelihood of the things that you don't like<01:20:33.199><c> um</c><01:20:34.080><c> all</c><01:20:34.280><c> the</c><01:20:34.440><c> rest</c><01:20:34.679><c> of</c><01:20:34.800><c> the</c><01:20:34.960><c> terms</c><01:20:35.560><c> here</c>"
+    },
+    {
+      "start": 4835.709,
+      "duration": 0.0,
+      "text": "like um all the rest of the terms here"
+    },
+    {
+      "start": 4835.719,
+      "duration": 0.0,
+      "text": "like um all the rest of the terms here it's<01:20:35.880><c> not</c><01:20:36.080><c> too</c><01:20:36.280><c> important</c><01:20:36.600><c> it's</c><01:20:36.800><c> actually</c>"
+    },
+    {
+      "start": 4837.51,
+      "duration": 0.0,
+      "text": "it's not too important it's actually"
+    },
+    {
+      "start": 4837.52,
+      "duration": 0.0,
+      "text": "it's not too important it's actually really<01:20:38.159><c> not</c><01:20:38.400><c> that</c><01:20:38.639><c> complicated</c><01:20:39.080><c> to</c>"
+    },
+    {
+      "start": 4839.47,
+      "duration": 0.0,
+      "text": "really not that complicated to"
+    },
+    {
+      "start": 4839.48,
+      "duration": 0.0,
+      "text": "really not that complicated to understand<01:20:39.920><c> but</c><01:20:40.040><c> at</c><01:20:40.159><c> a</c><01:20:40.280><c> high</c><01:20:40.440><c> level</c><01:20:40.920><c> it's</c>"
+    },
+    {
+      "start": 4841.07,
+      "duration": 0.0,
+      "text": "understand but at a high level it's"
+    },
+    {
+      "start": 4841.08,
+      "duration": 0.0,
+      "text": "understand but at a high level it's really<01:20:41.320><c> just</c><01:20:41.560><c> maximizing</c><01:20:42.400><c> the</c><01:20:42.520><c> things</c><01:20:42.679><c> you</c>"
+    },
+    {
+      "start": 4842.83,
+      "duration": 0.0,
+      "text": "really just maximizing the things you"
+    },
+    {
+      "start": 4842.84,
+      "duration": 0.0,
+      "text": "really just maximizing the things you like<01:20:43.120><c> minimizing</c><01:20:43.719><c> the</c><01:20:43.840><c> the</c><01:20:44.000><c> rest</c><01:20:44.800><c> um</c><01:20:45.800><c> and</c><01:20:46.639><c> one</c>"
+    },
+    {
+      "start": 4846.87,
+      "duration": 0.0,
+      "text": "like minimizing the the rest um and one"
+    },
+    {
+      "start": 4846.88,
+      "duration": 0.0,
+      "text": "like minimizing the the rest um and one thing<01:20:47.040><c> to</c><01:20:47.239><c> note</c><01:20:47.960><c> uh</c><01:20:48.120><c> which</c><01:20:48.239><c> I</c><01:20:48.320><c> was</c><01:20:48.440><c> going</c><01:20:48.560><c> to</c>"
+    },
+    {
+      "start": 4848.669,
+      "duration": 0.0,
+      "text": "thing to note uh which I was going to"
+    },
+    {
+      "start": 4848.679,
+      "duration": 0.0,
+      "text": "thing to note uh which I was going to say<01:20:48.840><c> just</c><01:20:49.280><c> here</c><01:20:49.679><c> is</c><01:20:49.800><c> that</c><01:20:50.000><c> actually</c><01:20:50.239><c> all</c><01:20:50.400><c> the</c>"
+    },
+    {
+      "start": 4850.51,
+      "duration": 0.0,
+      "text": "say just here is that actually all the"
+    },
+    {
+      "start": 4850.52,
+      "duration": 0.0,
+      "text": "say just here is that actually all the rest<01:20:50.760><c> is</c><01:20:50.920><c> chosen</c><01:20:51.520><c> such</c><01:20:51.920><c> that</c><01:20:52.760><c> um</c><01:20:53.080><c> the</c><01:20:53.239><c> global</c>"
+    },
+    {
+      "start": 4853.59,
+      "duration": 0.0,
+      "text": "rest is chosen such that um the global"
+    },
+    {
+      "start": 4853.6,
+      "duration": 0.0,
+      "text": "rest is chosen such that um the global Minima<01:20:54.440><c> of</c><01:20:54.679><c> of</c><01:20:55.040><c> Po</c><01:20:55.840><c> and</c><01:20:55.920><c> a</c><01:20:56.080><c> global</c><01:20:56.360><c> Minima</c><01:20:57.000><c> of</c>"
+    },
+    {
+      "start": 4857.189,
+      "duration": 0.0,
+      "text": "Minima of of Po and a global Minima of"
+    },
+    {
+      "start": 4857.199,
+      "duration": 0.0,
+      "text": "Minima of of Po and a global Minima of like<01:20:57.360><c> this</c><01:20:57.520><c> DPO</c><01:20:58.440><c> under</c><01:20:58.840><c> some</c><01:20:59.239><c> assumptions</c><01:20:59.760><c> are</c>"
+    },
+    {
+      "start": 4859.95,
+      "duration": 0.0,
+      "text": "like this DPO under some assumptions are"
+    },
+    {
+      "start": 4859.96,
+      "duration": 0.0,
+      "text": "like this DPO under some assumptions are essentially<01:21:00.360><c> equivalent</c><01:21:01.120><c> so</c><01:21:01.560><c> this</c><01:21:01.679><c> is</c><01:21:01.840><c> the</c>"
+    },
+    {
+      "start": 4862.07,
+      "duration": 0.0,
+      "text": "essentially equivalent so this is the"
+    },
+    {
+      "start": 4862.08,
+      "duration": 0.0,
+      "text": "essentially equivalent so this is the right<01:21:02.320><c> thing</c><01:21:02.719><c> to</c><01:21:02.880><c> do</c><01:21:03.600><c> mathematically</c><01:21:04.520><c> I'm</c><01:21:04.639><c> not</c>"
+    },
+    {
+      "start": 4864.79,
+      "duration": 0.0,
+      "text": "right thing to do mathematically I'm not"
+    },
+    {
+      "start": 4864.8,
+      "duration": 0.0,
+      "text": "right thing to do mathematically I'm not going<01:21:04.880><c> to</c><01:21:05.000><c> go</c><01:21:05.120><c> through</c><01:21:05.280><c> the</c><01:21:05.440><c> derivations</c><01:21:06.159><c> but</c>"
+    },
+    {
+      "start": 4866.75,
+      "duration": 0.0,
+      "text": "going to go through the derivations but"
+    },
+    {
+      "start": 4866.76,
+      "duration": 0.0,
+      "text": "going to go through the derivations but that's<01:21:06.920><c> the</c><01:21:07.040><c> right</c><01:21:07.199><c> thing</c><01:21:07.320><c> to</c><01:21:07.440><c> do</c><01:21:08.080><c> uh</c><01:21:08.199><c> it's</c>"
+    },
+    {
+      "start": 4868.39,
+      "duration": 0.0,
+      "text": "that's the right thing to do uh it's"
+    },
+    {
+      "start": 4868.4,
+      "duration": 0.0,
+      "text": "that's the right thing to do uh it's pretty<01:21:08.679><c> different</c><01:21:08.960><c> with</c><01:21:09.159><c> Po</c><01:21:09.520><c> in</c><01:21:09.600><c> the</c><01:21:09.719><c> sense</c>"
+    },
+    {
+      "start": 4869.95,
+      "duration": 0.0,
+      "text": "pretty different with Po in the sense"
+    },
+    {
+      "start": 4869.96,
+      "duration": 0.0,
+      "text": "pretty different with Po in the sense that<01:21:10.159><c> now</c><01:21:10.800><c> and</c><01:21:10.920><c> with</c><01:21:11.080><c> P</c><01:21:11.400><c> what</c><01:21:11.520><c> you</c><01:21:11.600><c> had</c><01:21:11.719><c> to</c><01:21:11.800><c> do</c>"
+    },
+    {
+      "start": 4871.95,
+      "duration": 0.0,
+      "text": "that now and with P what you had to do"
+    },
+    {
+      "start": 4871.96,
+      "duration": 0.0,
+      "text": "that now and with P what you had to do is<01:21:12.120><c> collect</c><01:21:12.400><c> the</c><01:21:12.520><c> human</c><01:21:12.760><c> preferences</c><01:21:13.560><c> then</c>"
+    },
+    {
+      "start": 4873.669,
+      "duration": 0.0,
+      "text": "is collect the human preferences then"
+    },
+    {
+      "start": 4873.679,
+      "duration": 0.0,
+      "text": "is collect the human preferences then train<01:21:13.920><c> a</c><01:21:14.280><c> uh</c><01:21:14.400><c> reward</c><01:21:14.719><c> model</c><01:21:15.040><c> with</c><01:21:15.159><c> maximum</c>"
+    },
+    {
+      "start": 4875.55,
+      "duration": 0.0,
+      "text": "train a uh reward model with maximum"
+    },
+    {
+      "start": 4875.56,
+      "duration": 0.0,
+      "text": "train a uh reward model with maximum likelihood<01:21:16.199><c> then</c><01:21:16.320><c> use</c><01:21:16.480><c> reinforcement</c>"
+    },
+    {
+      "start": 4876.99,
+      "duration": 0.0,
+      "text": "likelihood then use reinforcement"
+    },
+    {
+      "start": 4877.0,
+      "duration": 0.0,
+      "text": "likelihood then use reinforcement learning<01:21:17.560><c> now</c><01:21:17.760><c> all</c><01:21:17.880><c> you</c><01:21:18.000><c> do</c><01:21:18.199><c> is</c><01:21:18.320><c> basically</c>"
+    },
+    {
+      "start": 4878.669,
+      "duration": 0.0,
+      "text": "learning now all you do is basically"
+    },
+    {
+      "start": 4878.679,
+      "duration": 0.0,
+      "text": "learning now all you do is basically maximum<01:21:19.080><c> likelihood</c><01:21:19.800><c> much</c><01:21:20.000><c> simpler</c><01:21:20.679><c> yes</c><01:21:21.000><c> I</c>"
+    },
+    {
+      "start": 4881.07,
+      "duration": 0.0,
+      "text": "maximum likelihood much simpler yes I"
+    },
+    {
+      "start": 4881.08,
+      "duration": 0.0,
+      "text": "maximum likelihood much simpler yes I mean<01:21:21.320><c> yeah</c><01:21:21.480><c> so</c><01:21:21.600><c> it</c><01:21:21.719><c> seems</c><01:21:22.000><c> like</c><01:21:22.280><c> this</c><01:21:22.400><c> is</c><01:21:22.880><c> a</c>"
+    },
+    {
+      "start": 4883.149,
+      "duration": 0.0,
+      "text": "mean yeah so it seems like this is a"
+    },
+    {
+      "start": 4883.159,
+      "duration": 0.0,
+      "text": "mean yeah so it seems like this is a much<01:21:23.320><c> simpler</c><01:21:23.760><c> and</c><01:21:23.960><c> B</c><01:21:24.199><c> like</c><01:21:24.360><c> what</c><01:21:24.600><c> you</c><01:21:24.800><c> just</c>"
+    },
+    {
+      "start": 4885.03,
+      "duration": 0.0,
+      "text": "much simpler and B like what you just"
+    },
+    {
+      "start": 4885.04,
+      "duration": 0.0,
+      "text": "much simpler and B like what you just intuitively<01:21:25.880><c> do</c><01:21:26.159><c> if</c><01:21:26.719><c> this</c><01:21:27.320><c> why</c><01:21:27.520><c> did</c><01:21:28.000><c> they</c>"
+    },
+    {
+      "start": 4888.229,
+      "duration": 0.0,
+      "text": "intuitively do if this why did they"
+    },
+    {
+      "start": 4888.239,
+      "duration": 0.0,
+      "text": "intuitively do if this why did they start<01:21:28.600><c> with</c><01:21:28.840><c> this</c><01:21:29.000><c> reward</c><01:21:29.320><c> model</c><01:21:29.679><c> like</c><01:21:29.880><c> what</c>"
+    },
+    {
+      "start": 4890.39,
+      "duration": 0.0,
+      "text": "start with this reward model like what"
+    },
+    {
+      "start": 4890.4,
+      "duration": 0.0,
+      "text": "start with this reward model like what what<01:21:30.520><c> led</c><01:21:30.760><c> them</c><01:21:31.080><c> doing</c><01:21:31.440><c> that</c><01:21:31.800><c> I</c><01:21:31.880><c> think</c><01:21:32.000><c> it's</c><01:21:32.080><c> a</c>"
+    },
+    {
+      "start": 4892.189,
+      "duration": 0.0,
+      "text": "what led them doing that I think it's a"
+    },
+    {
+      "start": 4892.199,
+      "duration": 0.0,
+      "text": "what led them doing that I think it's a great<01:21:32.400><c> question</c><01:21:33.159><c> uh</c><01:21:33.280><c> I</c><01:21:33.360><c> don't</c><01:21:33.560><c> really</c><01:21:33.800><c> know</c>"
+    },
+    {
+      "start": 4894.51,
+      "duration": 0.0,
+      "text": "great question uh I don't really know"
+    },
+    {
+      "start": 4894.52,
+      "duration": 0.0,
+      "text": "great question uh I don't really know what<01:21:34.639><c> I</c><01:21:34.719><c> can</c><01:21:34.880><c> tell</c><01:21:35.080><c> you</c><01:21:35.320><c> is</c><01:21:35.480><c> that</c><01:21:35.880><c> at</c><01:21:36.120><c> open</c><01:21:36.520><c> ey</c>"
+    },
+    {
+      "start": 4897.149,
+      "duration": 0.0,
+      "text": "what I can tell you is that at open ey"
+    },
+    {
+      "start": 4897.159,
+      "duration": 0.0,
+      "text": "what I can tell you is that at open ey the<01:21:37.400><c> people</c><01:21:37.880><c> who</c><01:21:38.120><c> did</c><01:21:38.560><c> the</c><01:21:39.239><c> um</c><01:21:40.000><c> uh</c><01:21:40.120><c> who</c><01:21:40.320><c> did</c>"
+    },
+    {
+      "start": 4900.709,
+      "duration": 0.0,
+      "text": "the people who did the um uh who did"
+    },
+    {
+      "start": 4900.719,
+      "duration": 0.0,
+      "text": "the people who did the um uh who did basically<01:21:41.120><c> this</c><01:21:41.360><c> PP</c><01:21:41.719><c> uh</c><01:21:41.840><c> sorry</c><01:21:42.360><c> who</c><01:21:42.840><c> did</c><01:21:43.080><c> Chad</c>"
+    },
+    {
+      "start": 4903.35,
+      "duration": 0.0,
+      "text": "basically this PP uh sorry who did Chad"
+    },
+    {
+      "start": 4903.36,
+      "duration": 0.0,
+      "text": "basically this PP uh sorry who did Chad GPT<01:21:43.760><c> initially</c><01:21:44.600><c> are</c><01:21:44.840><c> the</c><01:21:45.000><c> ones</c><01:21:45.239><c> who</c><01:21:45.480><c> actually</c>"
+    },
+    {
+      "start": 4906.03,
+      "duration": 0.0,
+      "text": "GPT initially are the ones who actually"
+    },
+    {
+      "start": 4906.04,
+      "duration": 0.0,
+      "text": "GPT initially are the ones who actually wrote<01:21:46.480><c> Po</c><01:21:47.480><c> and</c><01:21:47.600><c> I</c><01:21:47.760><c> think</c><01:21:47.920><c> they</c><01:21:48.040><c> were</c><01:21:48.280><c> just</c><01:21:48.560><c> like</c>"
+    },
+    {
+      "start": 4908.669,
+      "duration": 0.0,
+      "text": "wrote Po and I think they were just like"
+    },
+    {
+      "start": 4908.679,
+      "duration": 0.0,
+      "text": "wrote Po and I think they were just like there<01:21:48.800><c> are</c><01:21:49.000><c> a</c><01:21:49.120><c> lot</c><01:21:49.239><c> of</c><01:21:49.360><c> reinforcement</c>"
+    },
+    {
+      "start": 4909.87,
+      "duration": 0.0,
+      "text": "there are a lot of reinforcement"
+    },
+    {
+      "start": 4909.88,
+      "duration": 0.0,
+      "text": "there are a lot of reinforcement learning<01:21:50.239><c> people</c><01:21:51.000><c> and</c><01:21:51.120><c> I</c><01:21:51.239><c> think</c><01:21:51.440><c> that</c><01:21:51.600><c> for</c>"
+    },
+    {
+      "start": 4911.79,
+      "duration": 0.0,
+      "text": "learning people and I think that for"
+    },
+    {
+      "start": 4911.8,
+      "duration": 0.0,
+      "text": "learning people and I think that for them<01:21:51.960><c> it</c><01:21:52.080><c> was</c><01:21:52.280><c> very</c><01:21:52.800><c> intuitive</c><01:21:53.800><c> um</c><01:21:54.760><c> so</c><01:21:55.520><c> there's</c>"
+    },
+    {
+      "start": 4915.75,
+      "duration": 0.0,
+      "text": "them it was very intuitive um so there's"
+    },
+    {
+      "start": 4915.76,
+      "duration": 0.0,
+      "text": "them it was very intuitive um so there's also<01:21:56.040><c> some</c><01:21:56.400><c> additional</c><01:21:56.960><c> like</c><01:21:57.159><c> potential</c>"
+    },
+    {
+      "start": 4917.629,
+      "duration": 0.0,
+      "text": "also some additional like potential"
+    },
+    {
+      "start": 4917.639,
+      "duration": 0.0,
+      "text": "also some additional like potential benefits<01:21:58.360><c> for</c><01:21:58.560><c> example</c><01:21:59.560><c> I</c><01:21:59.679><c> don't</c><01:21:59.800><c> want</c><01:22:00.000><c> to</c>"
+    },
+    {
+      "start": 4920.99,
+      "duration": 0.0,
+      "text": "benefits for example I don't want to"
+    },
+    {
+      "start": 4921.0,
+      "duration": 0.0,
+      "text": "benefits for example I don't want to yeah<01:22:01.159><c> for</c><01:22:01.320><c> example</c><01:22:01.560><c> if</c><01:22:01.639><c> you</c><01:22:01.760><c> use</c><01:22:01.920><c> the</c><01:22:02.040><c> reward</c>"
+    },
+    {
+      "start": 4922.39,
+      "duration": 0.0,
+      "text": "yeah for example if you use the reward"
+    },
+    {
+      "start": 4922.4,
+      "duration": 0.0,
+      "text": "yeah for example if you use the reward model<01:22:03.280><c> uh</c><01:22:03.400><c> the</c><01:22:03.520><c> cool</c><01:22:03.760><c> thing</c><01:22:03.920><c> here</c><01:22:04.080><c> with</c>"
+    },
+    {
+      "start": 4924.229,
+      "duration": 0.0,
+      "text": "model uh the cool thing here with"
+    },
+    {
+      "start": 4924.239,
+      "duration": 0.0,
+      "text": "model uh the cool thing here with reinforcement<01:22:04.760><c> learning</c><01:22:05.040><c> is</c><01:22:05.120><c> that</c><01:22:05.239><c> you</c><01:22:05.320><c> can</c>"
+    },
+    {
+      "start": 4925.43,
+      "duration": 0.0,
+      "text": "reinforcement learning is that you can"
+    },
+    {
+      "start": 4925.44,
+      "duration": 0.0,
+      "text": "reinforcement learning is that you can use<01:22:05.679><c> unlabeled</c><01:22:06.280><c> data</c><01:22:07.280><c> with</c><01:22:07.400><c> the</c><01:22:07.560><c> reward</c><01:22:07.920><c> model</c>"
+    },
+    {
+      "start": 4928.229,
+      "duration": 0.0,
+      "text": "use unlabeled data with the reward model"
+    },
+    {
+      "start": 4928.239,
+      "duration": 0.0,
+      "text": "use unlabeled data with the reward model so<01:22:08.480><c> here</c><01:22:08.600><c> you</c><01:22:08.719><c> can</c><01:22:08.920><c> only</c><01:22:09.159><c> use</c><01:22:09.360><c> the</c><01:22:09.520><c> label</c><01:22:09.880><c> data</c>"
+    },
+    {
+      "start": 4930.39,
+      "duration": 0.0,
+      "text": "so here you can only use the label data"
+    },
+    {
+      "start": 4930.4,
+      "duration": 0.0,
+      "text": "so here you can only use the label data for<01:22:10.639><c> doing</c><01:22:11.280><c> DPO</c><01:22:12.280><c> um</c><01:22:12.639><c> for</c><01:22:12.920><c> PP</c><01:22:13.360><c> for</c><01:22:13.520><c> po</c><01:22:13.880><c> you</c><01:22:13.960><c> first</c>"
+    },
+    {
+      "start": 4934.11,
+      "duration": 0.0,
+      "text": "for doing DPO um for PP for po you first"
+    },
+    {
+      "start": 4934.12,
+      "duration": 0.0,
+      "text": "for doing DPO um for PP for po you first train<01:22:14.400><c> your</c><01:22:14.520><c> reward</c><01:22:14.840><c> model</c><01:22:15.520><c> and</c><01:22:15.639><c> then</c><01:22:15.760><c> you</c><01:22:15.880><c> can</c>"
+    },
+    {
+      "start": 4935.99,
+      "duration": 0.0,
+      "text": "train your reward model and then you can"
+    },
+    {
+      "start": 4936.0,
+      "duration": 0.0,
+      "text": "train your reward model and then you can use<01:22:16.239><c> unlabeled</c><01:22:16.800><c> data</c><01:22:17.800><c> uh</c><01:22:18.280><c> where</c><01:22:18.440><c> the</c><01:22:18.600><c> reward</c>"
+    },
+    {
+      "start": 4938.87,
+      "duration": 0.0,
+      "text": "use unlabeled data uh where the reward"
+    },
+    {
+      "start": 4938.88,
+      "duration": 0.0,
+      "text": "use unlabeled data uh where the reward model<01:22:19.120><c> will</c><01:22:19.320><c> basically</c><01:22:19.719><c> label</c><01:22:20.280><c> this</c>"
+    },
+    {
+      "start": 4940.47,
+      "duration": 0.0,
+      "text": "model will basically label this"
+    },
+    {
+      "start": 4940.48,
+      "duration": 0.0,
+      "text": "model will basically label this unlabeled<01:22:20.960><c> data</c><01:22:21.280><c> so</c><01:22:21.480><c> there</c><01:22:21.719><c> there's</c>"
+    },
+    {
+      "start": 4941.91,
+      "duration": 0.0,
+      "text": "unlabeled data so there there's"
+    },
+    {
+      "start": 4941.92,
+      "duration": 0.0,
+      "text": "unlabeled data so there there's additional<01:22:22.360><c> kind</c><01:22:22.520><c> of</c><01:22:23.320><c> potential</c><01:22:24.320><c> uh</c>"
+    },
+    {
+      "start": 4945.43,
+      "duration": 0.0,
+      "text": "additional kind of potential uh"
+    },
+    {
+      "start": 4945.44,
+      "duration": 0.0,
+      "text": "additional kind of potential uh there<01:22:25.600><c> could</c><01:22:25.760><c> be</c><01:22:25.960><c> potential</c><01:22:26.320><c> improvements</c><01:22:27.080><c> in</c>"
+    },
+    {
+      "start": 4947.27,
+      "duration": 0.0,
+      "text": "there could be potential improvements in"
+    },
+    {
+      "start": 4947.28,
+      "duration": 0.0,
+      "text": "there could be potential improvements in practice<01:22:27.600><c> it</c><01:22:27.800><c> happens</c><01:22:28.120><c> at</c><01:22:28.400><c> down</c><01:22:28.639><c> and</c><01:22:28.760><c> on</c><01:22:29.360><c> and</c><01:22:29.520><c> I</c>"
+    },
+    {
+      "start": 4949.629,
+      "duration": 0.0,
+      "text": "practice it happens at down and on and I"
+    },
+    {
+      "start": 4949.639,
+      "duration": 0.0,
+      "text": "practice it happens at down and on and I think<01:22:30.400><c> just</c><01:22:30.600><c> that</c><01:22:30.760><c> a</c><01:22:30.840><c> lot</c><01:22:30.960><c> of</c><01:22:31.120><c> people</c><01:22:31.679><c> in</c><01:22:31.840><c> this</c>"
+    },
+    {
+      "start": 4952.03,
+      "duration": 0.0,
+      "text": "think just that a lot of people in this"
+    },
+    {
+      "start": 4952.04,
+      "duration": 0.0,
+      "text": "think just that a lot of people in this team<01:22:32.520><c> were</c><01:22:32.719><c> reinforcement</c><01:22:33.280><c> learning</c><01:22:33.639><c> experts</c>"
+    },
+    {
+      "start": 4954.39,
+      "duration": 0.0,
+      "text": "team were reinforcement learning experts"
+    },
+    {
+      "start": 4954.4,
+      "duration": 0.0,
+      "text": "team were reinforcement learning experts including<01:22:35.040><c> uh</c><01:22:35.159><c> the</c><01:22:35.239><c> main</c><01:22:35.440><c> author</c><01:22:35.679><c> of</c><01:22:35.800><c> Po</c><01:22:36.360><c> John</c>"
+    },
+    {
+      "start": 4957.87,
+      "duration": 0.0,
+      "text": "including uh the main author of Po John"
+    },
+    {
+      "start": 4957.88,
+      "duration": 0.0,
+      "text": "including uh the main author of Po John hman<01:22:38.880><c> um</c><01:22:39.280><c> so</c><01:22:39.600><c> much</c><01:22:39.800><c> simpler</c><01:22:40.120><c> in</c><01:22:40.239><c> poo</c><01:22:41.000><c> and</c><01:22:41.120><c> is</c>"
+    },
+    {
+      "start": 4961.43,
+      "duration": 0.0,
+      "text": "hman um so much simpler in poo and is"
+    },
+    {
+      "start": 4961.44,
+      "duration": 0.0,
+      "text": "hman um so much simpler in poo and is basically<01:22:41.800><c> performs</c><01:22:42.239><c> as</c><01:22:42.400><c> well</c><01:22:43.000><c> uh</c><01:22:43.120><c> so</c><01:22:43.320><c> now</c>"
+    },
+    {
+      "start": 4963.55,
+      "duration": 0.0,
+      "text": "basically performs as well uh so now"
+    },
+    {
+      "start": 4963.56,
+      "duration": 0.0,
+      "text": "basically performs as well uh so now this<01:22:43.679><c> is</c><01:22:43.800><c> the</c><01:22:43.960><c> standard</c><01:22:44.760><c> uh</c><01:22:44.920><c> thing</c><01:22:45.040><c> that</c>"
+    },
+    {
+      "start": 4965.189,
+      "duration": 0.0,
+      "text": "this is the standard uh thing that"
+    },
+    {
+      "start": 4965.199,
+      "duration": 0.0,
+      "text": "this is the standard uh thing that people<01:22:45.480><c> use</c><01:22:46.280><c> at</c><01:22:46.440><c> least</c><01:22:46.639><c> in</c><01:22:46.719><c> the</c><01:22:46.840><c> open</c><01:22:47.040><c> source</c>"
+    },
+    {
+      "start": 4967.27,
+      "duration": 0.0,
+      "text": "people use at least in the open source"
+    },
+    {
+      "start": 4967.28,
+      "duration": 0.0,
+      "text": "people use at least in the open source Community<01:22:47.960><c> I</c><01:22:48.080><c> believe</c><01:22:48.400><c> it's</c><01:22:48.600><c> actually</c><01:22:48.800><c> the</c>"
+    },
+    {
+      "start": 4968.91,
+      "duration": 0.0,
+      "text": "Community I believe it's actually the"
+    },
+    {
+      "start": 4968.92,
+      "duration": 0.0,
+      "text": "Community I believe it's actually the standard<01:22:49.320><c> also</c><01:22:49.520><c> in</c><01:22:50.239><c> in</c><01:22:51.120><c> Industry</c><01:22:52.120><c> so</c><01:22:52.280><c> that's</c>"
+    },
+    {
+      "start": 4972.59,
+      "duration": 0.0,
+      "text": "standard also in in Industry so that's"
+    },
+    {
+      "start": 4972.6,
+      "duration": 0.0,
+      "text": "standard also in in Industry so that's called<01:22:53.080><c> DPO</c><01:22:54.080><c> gains</c>"
+    },
+    {
+      "start": 4975.07,
+      "duration": 0.0,
+      "text": "called DPO gains"
+    },
+    {
+      "start": 4975.08,
+      "duration": 0.0,
+      "text": "called DPO gains um<01:22:55.320><c> so</c><01:22:55.560><c> those</c><01:22:55.679><c> are</c><01:22:56.080><c> all</c><01:22:56.360><c> the</c><01:22:56.600><c> papers</c><01:22:57.040><c> on</c><01:22:57.159><c> the</c>"
+    },
+    {
+      "start": 4977.35,
+      "duration": 0.0,
+      "text": "um so those are all the papers on the"
+    },
+    {
+      "start": 4977.36,
+      "duration": 0.0,
+      "text": "um so those are all the papers on the left<01:22:57.719><c> here</c><01:22:57.920><c> this</c><01:22:58.000><c> is</c><01:22:58.120><c> on</c><01:22:58.239><c> a</c><01:22:58.360><c> summarization</c>"
+    },
+    {
+      "start": 4979.07,
+      "duration": 0.0,
+      "text": "left here this is on a summarization"
+    },
+    {
+      "start": 4979.08,
+      "duration": 0.0,
+      "text": "left here this is on a summarization task<01:22:59.639><c> you</c><01:22:59.800><c> see</c><01:23:00.400><c> all</c><01:23:00.560><c> I</c><01:23:00.679><c> want</c><01:23:00.760><c> to</c><01:23:00.920><c> show</c><01:23:01.120><c> you</c><01:23:01.400><c> is</c>"
+    },
+    {
+      "start": 4981.55,
+      "duration": 0.0,
+      "text": "task you see all I want to show you is"
+    },
+    {
+      "start": 4981.56,
+      "duration": 0.0,
+      "text": "task you see all I want to show you is that<01:23:01.760><c> basically</c><01:23:02.040><c> the</c><01:23:02.159><c> pre-train</c><01:23:02.679><c> models</c><01:23:03.679><c> uh</c>"
+    },
+    {
+      "start": 4983.83,
+      "duration": 0.0,
+      "text": "that basically the pre-train models uh"
+    },
+    {
+      "start": 4983.84,
+      "duration": 0.0,
+      "text": "that basically the pre-train models uh were<01:23:04.239><c> okay</c><01:23:04.560><c> and</c><01:23:04.679><c> they</c><01:23:04.800><c> improve</c><01:23:05.120><c> with</c><01:23:05.280><c> scale</c><01:23:05.880><c> if</c>"
+    },
+    {
+      "start": 4985.95,
+      "duration": 0.0,
+      "text": "were okay and they improve with scale if"
+    },
+    {
+      "start": 4985.96,
+      "duration": 0.0,
+      "text": "were okay and they improve with scale if you<01:23:06.080><c> do</c><01:23:06.199><c> supervised</c><01:23:06.679><c> fine</c><01:23:06.880><c> tuning</c><01:23:07.239><c> you</c>"
+    },
+    {
+      "start": 4987.39,
+      "duration": 0.0,
+      "text": "you do supervised fine tuning you"
+    },
+    {
+      "start": 4987.4,
+      "duration": 0.0,
+      "text": "you do supervised fine tuning you improve<01:23:07.719><c> them</c><01:23:07.880><c> a</c><01:23:07.960><c> little</c><01:23:08.120><c> bit</c><01:23:08.280><c> more</c><01:23:08.760><c> if</c><01:23:08.880><c> you</c><01:23:09.040><c> do</c>"
+    },
+    {
+      "start": 4989.39,
+      "duration": 0.0,
+      "text": "improve them a little bit more if you do"
+    },
+    {
+      "start": 4989.4,
+      "duration": 0.0,
+      "text": "improve them a little bit more if you do po<01:23:10.040><c> or</c><01:23:10.239><c> something</c><01:23:10.639><c> with</c><01:23:10.800><c> all</c><01:23:11.040><c> HF</c><01:23:11.400><c> with</c><01:23:11.520><c> human</c>"
+    },
+    {
+      "start": 4991.75,
+      "duration": 0.0,
+      "text": "po or something with all HF with human"
+    },
+    {
+      "start": 4991.76,
+      "duration": 0.0,
+      "text": "po or something with all HF with human feedback<01:23:12.360><c> you</c><01:23:12.560><c> get</c><01:23:12.880><c> performance</c><01:23:13.320><c> that</c><01:23:13.480><c> are</c><01:23:14.120><c> as</c>"
+    },
+    {
+      "start": 4994.75,
+      "duration": 0.0,
+      "text": "feedback you get performance that are as"
+    },
+    {
+      "start": 4994.76,
+      "duration": 0.0,
+      "text": "feedback you get performance that are as often<01:23:15.080><c> times</c><01:23:15.719><c> depending</c><01:23:16.040><c> on</c><01:23:16.159><c> a</c><01:23:16.280><c> benchmark</c>"
+    },
+    {
+      "start": 4996.87,
+      "duration": 0.0,
+      "text": "often times depending on a benchmark"
+    },
+    {
+      "start": 4996.88,
+      "duration": 0.0,
+      "text": "often times depending on a benchmark even<01:23:17.120><c> better</c><01:23:17.400><c> than</c><01:23:17.960><c> uh</c><01:23:18.120><c> humans</c><01:23:18.520><c> so</c><01:23:18.719><c> this</c><01:23:18.840><c> is</c>"
+    },
+    {
+      "start": 4998.99,
+      "duration": 0.0,
+      "text": "even better than uh humans so this is"
+    },
+    {
+      "start": 4999.0,
+      "duration": 0.0,
+      "text": "even better than uh humans so this is the<01:23:19.199><c> human</c><01:23:19.960><c> uh</c><01:23:20.120><c> reference</c><01:23:20.600><c> summaries</c><01:23:21.600><c> same</c>"
+    },
+    {
+      "start": 5001.79,
+      "duration": 0.0,
+      "text": "the human uh reference summaries same"
+    },
+    {
+      "start": 5001.8,
+      "duration": 0.0,
+      "text": "the human uh reference summaries same thing<01:23:21.960><c> this</c><01:23:22.080><c> is</c><01:23:22.159><c> on</c><01:23:22.320><c> a</c><01:23:22.719><c> uh</c><01:23:22.960><c> on</c><01:23:23.080><c> a</c><01:23:23.199><c> paper</c><01:23:23.480><c> that</c><01:23:23.600><c> we</c>"
+    },
+    {
+      "start": 5003.669,
+      "duration": 0.0,
+      "text": "thing this is on a uh on a paper that we"
+    },
+    {
+      "start": 5003.679,
+      "duration": 0.0,
+      "text": "thing this is on a uh on a paper that we have<01:23:23.840><c> Alpaca</c><01:23:24.239><c> Farm</c>"
+    },
+    {
+      "start": 5005.149,
+      "duration": 0.0,
+      "text": "have Alpaca Farm"
+    },
+    {
+      "start": 5005.159,
+      "duration": 0.0,
+      "text": "have Alpaca Farm where<01:23:25.280><c> we</c><01:23:25.400><c> see</c><01:23:26.199><c> uh</c><01:23:26.320><c> the</c><01:23:26.440><c> evaluation</c><01:23:26.880><c> here</c><01:23:27.000><c> is</c>"
+    },
+    {
+      "start": 5007.07,
+      "duration": 0.0,
+      "text": "where we see uh the evaluation here is"
+    },
+    {
+      "start": 5007.08,
+      "duration": 0.0,
+      "text": "where we see uh the evaluation here is not<01:23:27.199><c> too</c><01:23:27.400><c> important</c><01:23:27.760><c> but</c><01:23:27.920><c> basically</c><01:23:28.199><c> you</c><01:23:28.280><c> see</c>"
+    },
+    {
+      "start": 5008.47,
+      "duration": 0.0,
+      "text": "not too important but basically you see"
+    },
+    {
+      "start": 5008.48,
+      "duration": 0.0,
+      "text": "not too important but basically you see pre-train<01:23:28.880><c> model</c><01:23:29.440><c> you</c><01:23:29.639><c> jump</c><01:23:29.920><c> to</c><01:23:30.520><c> sft</c><01:23:31.400><c> and</c><01:23:31.520><c> then</c>"
+    },
+    {
+      "start": 5011.59,
+      "duration": 0.0,
+      "text": "pre-train model you jump to sft and then"
+    },
+    {
+      "start": 5011.6,
+      "duration": 0.0,
+      "text": "pre-train model you jump to sft and then you<01:23:31.760><c> jump</c><01:23:31.960><c> to</c><01:23:32.080><c> PPO</c><01:23:32.920><c> and</c><01:23:33.080><c> popo</c><01:23:33.800><c> have</c><01:23:33.960><c> the</c><01:23:34.080><c> exact</c>"
+    },
+    {
+      "start": 5014.35,
+      "duration": 0.0,
+      "text": "you jump to PPO and popo have the exact"
+    },
+    {
+      "start": 5014.36,
+      "duration": 0.0,
+      "text": "you jump to PPO and popo have the exact same"
+    },
+    {
+      "start": 5015.83,
+      "duration": 0.0,
+      "text": "same"
+    },
+    {
+      "start": 5015.84,
+      "duration": 0.0,
+      "text": "same performance<01:23:36.840><c> so</c><01:23:37.159><c> basically</c><01:23:37.520><c> all</c><01:23:37.800><c> HF</c><01:23:38.159><c> helps</c>"
+    },
+    {
+      "start": 5018.83,
+      "duration": 0.0,
+      "text": "performance so basically all HF helps"
+    },
+    {
+      "start": 5018.84,
+      "duration": 0.0,
+      "text": "performance so basically all HF helps that's<01:23:39.040><c> kind</c><01:23:39.159><c> of</c><01:23:39.280><c> the</c><01:23:39.400><c> conclusion</c><01:23:40.120><c> and</c><01:23:40.280><c> DPO</c><01:23:40.719><c> is</c>"
+    },
+    {
+      "start": 5021.39,
+      "duration": 0.0,
+      "text": "that's kind of the conclusion and DPO is"
+    },
+    {
+      "start": 5021.4,
+      "duration": 0.0,
+      "text": "that's kind of the conclusion and DPO is simple<01:23:42.400><c> uh</c><01:23:42.600><c> data</c><01:23:43.400><c> uh</c><01:23:43.560><c> the</c><01:23:43.800><c> way</c><01:23:44.040><c> that</c><01:23:44.159><c> you</c>"
+    },
+    {
+      "start": 5024.35,
+      "duration": 0.0,
+      "text": "simple uh data uh the way that you"
+    },
+    {
+      "start": 5024.36,
+      "duration": 0.0,
+      "text": "simple uh data uh the way that you collect<01:23:44.719><c> that</c><01:23:44.880><c> type</c><01:23:45.040><c> of</c><01:23:45.239><c> data</c><01:23:46.120><c> um</c><01:23:47.080><c> first</c><01:23:47.400><c> idea</c>"
+    },
+    {
+      "start": 5027.87,
+      "duration": 0.0,
+      "text": "collect that type of data um first idea"
+    },
+    {
+      "start": 5027.88,
+      "duration": 0.0,
+      "text": "collect that type of data um first idea is<01:23:48.080><c> just</c><01:23:48.239><c> use</c><01:23:48.679><c> humans</c><01:23:49.239><c> as</c><01:23:49.360><c> we</c><01:23:49.520><c> already</c><01:23:49.760><c> talked</c>"
+    },
+    {
+      "start": 5030.03,
+      "duration": 0.0,
+      "text": "is just use humans as we already talked"
+    },
+    {
+      "start": 5030.04,
+      "duration": 0.0,
+      "text": "is just use humans as we already talked about<01:23:50.840><c> uh</c><01:23:51.080><c> guidelines</c><01:23:51.480><c> are</c><01:23:51.679><c> very</c><01:23:51.880><c> complicated</c>"
+    },
+    {
+      "start": 5032.709,
+      "duration": 0.0,
+      "text": "about uh guidelines are very complicated"
+    },
+    {
+      "start": 5032.719,
+      "duration": 0.0,
+      "text": "about uh guidelines are very complicated for<01:23:52.920><c> what</c><01:23:53.080><c> humans</c><01:23:53.360><c> should</c><01:23:53.520><c> be</c><01:23:53.639><c> labeling</c><01:23:54.280><c> and</c>"
+    },
+    {
+      "start": 5034.43,
+      "duration": 0.0,
+      "text": "for what humans should be labeling and"
+    },
+    {
+      "start": 5034.44,
+      "duration": 0.0,
+      "text": "for what humans should be labeling and and<01:23:54.480><c> it's</c><01:23:54.639><c> really</c><01:23:54.840><c> not</c><01:23:55.040><c> that</c><01:23:55.199><c> easy</c><01:23:55.679><c> and</c>"
+    },
+    {
+      "start": 5035.87,
+      "duration": 0.0,
+      "text": "and it's really not that easy and"
+    },
+    {
+      "start": 5035.88,
+      "duration": 0.0,
+      "text": "and it's really not that easy and actually<01:23:56.120><c> if</c><01:23:56.199><c> you</c><01:23:56.400><c> ever</c><01:23:56.639><c> do</c><01:23:56.920><c> some</c><01:23:57.120><c> of</c><01:23:57.239><c> the</c>"
+    },
+    {
+      "start": 5037.35,
+      "duration": 0.0,
+      "text": "actually if you ever do some of the"
+    },
+    {
+      "start": 5037.36,
+      "duration": 0.0,
+      "text": "actually if you ever do some of the labeling<01:23:58.159><c> you</c><01:23:58.239><c> will</c><01:23:58.480><c> see</c><01:23:58.840><c> that</c><01:23:59.480><c> it's</c>"
+    },
+    {
+      "start": 5040.03,
+      "duration": 0.0,
+      "text": "labeling you will see that it's"
+    },
+    {
+      "start": 5040.04,
+      "duration": 0.0,
+      "text": "labeling you will see that it's extremely<01:24:00.639><c> complicated</c><01:24:01.400><c> like</c><01:24:01.520><c> if</c><01:24:01.639><c> I</c><01:24:01.800><c> zoom</c><01:24:02.080><c> in</c>"
+    },
+    {
+      "start": 5042.35,
+      "duration": 0.0,
+      "text": "extremely complicated like if I zoom in"
+    },
+    {
+      "start": 5042.36,
+      "duration": 0.0,
+      "text": "extremely complicated like if I zoom in to<01:24:02.719><c> this</c><01:24:03.719><c> uh</c><01:24:03.960><c> here</c><01:24:04.120><c> I</c><01:24:04.239><c> have</c><01:24:04.360><c> a</c><01:24:04.520><c> question</c><01:24:05.080><c> tell</c>"
+    },
+    {
+      "start": 5045.59,
+      "duration": 0.0,
+      "text": "to this uh here I have a question tell"
+    },
+    {
+      "start": 5045.6,
+      "duration": 0.0,
+      "text": "to this uh here I have a question tell tell<01:24:06.000><c> me</c><01:24:06.239><c> about</c><01:24:06.440><c> self-driving</c><01:24:07.000><c> cars</c><01:24:07.719><c> and</c><01:24:07.800><c> you</c>"
+    },
+    {
+      "start": 5047.91,
+      "duration": 0.0,
+      "text": "tell me about self-driving cars and you"
+    },
+    {
+      "start": 5047.92,
+      "duration": 0.0,
+      "text": "tell me about self-driving cars and you read<01:24:08.159><c> both</c><01:24:08.360><c> self-driving</c><01:24:08.880><c> cars</c><01:24:09.080><c> are</c><01:24:09.239><c> vehicles</c>"
+    },
+    {
+      "start": 5049.55,
+      "duration": 0.0,
+      "text": "read both self-driving cars are vehicles"
+    },
+    {
+      "start": 5049.56,
+      "duration": 0.0,
+      "text": "read both self-driving cars are vehicles that<01:24:09.679><c> are</c><01:24:09.800><c> capable</c><01:24:10.080><c> of</c><01:24:10.199><c> detecting</c><01:24:10.600><c> their</c>"
+    },
+    {
+      "start": 5050.75,
+      "duration": 0.0,
+      "text": "that are capable of detecting their"
+    },
+    {
+      "start": 5050.76,
+      "duration": 0.0,
+      "text": "that are capable of detecting their surroundings<01:24:11.239><c> blah</c><01:24:11.400><c> blah</c><01:24:11.600><c> blah</c><01:24:11.960><c> self-driving</c>"
+    },
+    {
+      "start": 5052.51,
+      "duration": 0.0,
+      "text": "surroundings blah blah blah self-driving"
+    },
+    {
+      "start": 5052.52,
+      "duration": 0.0,
+      "text": "surroundings blah blah blah self-driving cars<01:24:12.719><c> are</c><01:24:12.920><c> cars</c><01:24:13.120><c> that</c><01:24:13.239><c> are</c><01:24:13.360><c> equipped</c><01:24:13.679><c> with</c>"
+    },
+    {
+      "start": 5053.79,
+      "duration": 0.0,
+      "text": "cars are cars that are equipped with"
+    },
+    {
+      "start": 5053.8,
+      "duration": 0.0,
+      "text": "cars are cars that are equipped with sensors<01:24:14.199><c> blah</c><01:24:14.400><c> blah</c><01:24:14.600><c> blah</c><01:24:14.880><c> to</c><01:24:15.040><c> navigate</c>"
+    },
+    {
+      "start": 5055.43,
+      "duration": 0.0,
+      "text": "sensors blah blah blah to navigate"
+    },
+    {
+      "start": 5055.44,
+      "duration": 0.0,
+      "text": "sensors blah blah blah to navigate without<01:24:15.679><c> the</c><01:24:15.800><c> need</c><01:24:15.920><c> for</c><01:24:16.080><c> a</c><01:24:16.199><c> driver</c><01:24:16.679><c> I</c><01:24:16.760><c> mean</c>"
+    },
+    {
+      "start": 5056.99,
+      "duration": 0.0,
+      "text": "without the need for a driver I mean"
+    },
+    {
+      "start": 5057.0,
+      "duration": 0.0,
+      "text": "without the need for a driver I mean both<01:24:17.199><c> seem</c><01:24:17.639><c> okay</c><01:24:18.120><c> like</c><01:24:18.280><c> which</c><01:24:18.440><c> one</c><01:24:18.600><c> is</c><01:24:18.719><c> better</c>"
+    },
+    {
+      "start": 5059.189,
+      "duration": 0.0,
+      "text": "both seem okay like which one is better"
+    },
+    {
+      "start": 5059.199,
+      "duration": 0.0,
+      "text": "both seem okay like which one is better it's<01:24:19.400><c> actually</c><01:24:19.600><c> hard</c><01:24:19.800><c> to</c><01:24:19.960><c> say</c><01:24:20.280><c> at</c><01:24:20.400><c> a</c><01:24:20.600><c> glance</c><01:24:21.400><c> um</c>"
+    },
+    {
+      "start": 5061.99,
+      "duration": 0.0,
+      "text": "it's actually hard to say at a glance um"
+    },
+    {
+      "start": 5062.0,
+      "duration": 0.0,
+      "text": "it's actually hard to say at a glance um and<01:24:22.159><c> as</c><01:24:22.239><c> a</c><01:24:22.440><c> result</c><01:24:23.080><c> uh</c><01:24:23.199><c> the</c><01:24:23.400><c> problem</c><01:24:23.639><c> with</c>"
+    },
+    {
+      "start": 5063.79,
+      "duration": 0.0,
+      "text": "and as a result uh the problem with"
+    },
+    {
+      "start": 5063.8,
+      "duration": 0.0,
+      "text": "and as a result uh the problem with humans<01:24:24.639><c> is</c><01:24:24.920><c> that</c><01:24:25.840><c> you</c><01:24:25.960><c> will</c><01:24:26.159><c> start</c><01:24:26.520><c> optimizing</c>"
+    },
+    {
+      "start": 5067.07,
+      "duration": 0.0,
+      "text": "humans is that you will start optimizing"
+    },
+    {
+      "start": 5067.08,
+      "duration": 0.0,
+      "text": "humans is that you will start optimizing a<01:24:27.199><c> lot</c><01:24:27.320><c> of</c><01:24:27.480><c> like</c><01:24:27.639><c> high</c><01:24:27.840><c> level</c><01:24:28.080><c> features</c><01:24:28.520><c> for</c>"
+    },
+    {
+      "start": 5068.669,
+      "duration": 0.0,
+      "text": "a lot of like high level features for"
+    },
+    {
+      "start": 5068.679,
+      "duration": 0.0,
+      "text": "a lot of like high level features for example<01:24:29.000><c> the</c><01:24:29.080><c> second</c><01:24:29.320><c> one</c><01:24:29.440><c> is</c><01:24:29.600><c> longer</c><01:24:30.199><c> I</c><01:24:30.320><c> can</c>"
+    },
+    {
+      "start": 5070.709,
+      "duration": 0.0,
+      "text": "example the second one is longer I can"
+    },
+    {
+      "start": 5070.719,
+      "duration": 0.0,
+      "text": "example the second one is longer I can guarantee<01:24:31.120><c> you</c><01:24:31.280><c> that</c><01:24:31.400><c> most</c><01:24:31.560><c> humans</c><01:24:31.840><c> will</c>"
+    },
+    {
+      "start": 5071.95,
+      "duration": 0.0,
+      "text": "guarantee you that most humans will"
+    },
+    {
+      "start": 5071.96,
+      "duration": 0.0,
+      "text": "guarantee you that most humans will choose<01:24:32.520><c> second</c><01:24:32.800><c> one</c><01:24:33.400><c> even</c><01:24:33.719><c> though</c><01:24:34.440><c> I</c><01:24:34.480><c> mean</c>"
+    },
+    {
+      "start": 5074.629,
+      "duration": 0.0,
+      "text": "choose second one even though I mean"
+    },
+    {
+      "start": 5074.639,
+      "duration": 0.0,
+      "text": "choose second one even though I mean maybe<01:24:34.840><c> the</c><01:24:34.960><c> first</c><01:24:35.159><c> one</c><01:24:35.280><c> is</c><01:24:35.400><c> better</c><01:24:35.639><c> I</c><01:24:35.679><c> don't</c>"
+    },
+    {
+      "start": 5075.83,
+      "duration": 0.0,
+      "text": "maybe the first one is better I don't"
+    },
+    {
+      "start": 5075.84,
+      "duration": 0.0,
+      "text": "maybe the first one is better I don't know<01:24:36.320><c> I</c><01:24:36.440><c> haven't</c><01:24:36.719><c> read</c><01:24:36.880><c> it</c><01:24:37.520><c> carefully</c><01:24:38.520><c> so</c>"
+    },
+    {
+      "start": 5078.669,
+      "duration": 0.0,
+      "text": "know I haven't read it carefully so"
+    },
+    {
+      "start": 5078.679,
+      "duration": 0.0,
+      "text": "know I haven't read it carefully so challenges<01:24:39.080><c> with</c><01:24:39.280><c> humans</c><01:24:40.000><c> first</c><01:24:40.480><c> slow</c><01:24:40.760><c> and</c>"
+    },
+    {
+      "start": 5081.35,
+      "duration": 0.0,
+      "text": "challenges with humans first slow and"
+    },
+    {
+      "start": 5081.36,
+      "duration": 0.0,
+      "text": "challenges with humans first slow and expensive<01:24:42.360><c> uh</c><01:24:42.719><c> second</c><01:24:43.360><c> as</c><01:24:43.520><c> I</c><01:24:43.679><c> just</c><01:24:43.880><c> mentioned</c>"
+    },
+    {
+      "start": 5084.55,
+      "duration": 0.0,
+      "text": "expensive uh second as I just mentioned"
+    },
+    {
+      "start": 5084.56,
+      "duration": 0.0,
+      "text": "expensive uh second as I just mentioned it's<01:24:44.760><c> hard</c><01:24:44.960><c> to</c><01:24:45.199><c> focus</c><01:24:45.639><c> on</c><01:24:45.840><c> things</c><01:24:46.040><c> that</c><01:24:46.199><c> matter</c>"
+    },
+    {
+      "start": 5086.55,
+      "duration": 0.0,
+      "text": "it's hard to focus on things that matter"
+    },
+    {
+      "start": 5086.56,
+      "duration": 0.0,
+      "text": "it's hard to focus on things that matter like<01:24:46.719><c> correctness</c><01:24:47.440><c> and</c><01:24:47.639><c> people</c><01:24:48.480><c> uh</c><01:24:48.639><c> usually</c>"
+    },
+    {
+      "start": 5088.99,
+      "duration": 0.0,
+      "text": "like correctness and people uh usually"
+    },
+    {
+      "start": 5089.0,
+      "duration": 0.0,
+      "text": "like correctness and people uh usually look<01:24:49.199><c> at</c><01:24:49.400><c> things</c><01:24:49.639><c> that</c><01:24:50.239><c> don't</c><01:24:50.480><c> matter</c><01:24:50.760><c> as</c><01:24:50.880><c> much</c>"
+    },
+    {
+      "start": 5091.149,
+      "duration": 0.0,
+      "text": "look at things that don't matter as much"
+    },
+    {
+      "start": 5091.159,
+      "duration": 0.0,
+      "text": "look at things that don't matter as much like<01:24:51.320><c> the</c><01:24:51.480><c> form</c><01:24:51.920><c> like</c><01:24:52.520><c> length</c><01:24:53.520><c> uh</c><01:24:53.639><c> and</c><01:24:53.760><c> as</c><01:24:53.840><c> a</c>"
+    },
+    {
+      "start": 5093.99,
+      "duration": 0.0,
+      "text": "like the form like length uh and as a"
+    },
+    {
+      "start": 5094.0,
+      "duration": 0.0,
+      "text": "like the form like length uh and as a result<01:24:54.480><c> so</c><01:24:54.639><c> what</c><01:24:54.719><c> I</c><01:24:54.840><c> show</c><01:24:55.080><c> here</c><01:24:55.239><c> is</c><01:24:55.360><c> that</c><01:24:55.760><c> uh</c>"
+    },
+    {
+      "start": 5095.87,
+      "duration": 0.0,
+      "text": "result so what I show here is that uh"
+    },
+    {
+      "start": 5095.88,
+      "duration": 0.0,
+      "text": "result so what I show here is that uh when<01:24:56.000><c> you</c><01:24:56.080><c> do</c><01:24:56.239><c> lhf</c><01:24:57.000><c> the</c><01:24:57.080><c> more</c><01:24:57.239><c> you</c><01:24:57.360><c> do</c><01:24:57.600><c> of</c><01:24:57.760><c> lhf</c>"
+    },
+    {
+      "start": 5098.35,
+      "duration": 0.0,
+      "text": "when you do lhf the more you do of lhf"
+    },
+    {
+      "start": 5098.36,
+      "duration": 0.0,
+      "text": "when you do lhf the more you do of lhf the<01:24:58.480><c> longer</c><01:24:58.760><c> the</c><01:24:58.840><c> output</c><01:24:59.159><c> of</c><01:24:59.320><c> the</c><01:24:59.679><c> of</c><01:24:59.960><c> the</c>"
+    },
+    {
+      "start": 5100.47,
+      "duration": 0.0,
+      "text": "the longer the output of the of the"
+    },
+    {
+      "start": 5100.48,
+      "duration": 0.0,
+      "text": "the longer the output of the of the models<01:25:00.920><c> become</c><01:25:01.400><c> so</c><01:25:01.560><c> if</c><01:25:01.639><c> you've</c><01:25:01.800><c> ever</c><01:25:01.960><c> been</c>"
+    },
+    {
+      "start": 5102.149,
+      "duration": 0.0,
+      "text": "models become so if you've ever been"
+    },
+    {
+      "start": 5102.159,
+      "duration": 0.0,
+      "text": "models become so if you've ever been annoyed<01:25:02.760><c> at</c><01:25:02.960><c> chat</c><01:25:03.159><c> GPT</c><01:25:03.600><c> answering</c><01:25:04.000><c> you</c><01:25:04.280><c> super</c>"
+    },
+    {
+      "start": 5104.51,
+      "duration": 0.0,
+      "text": "annoyed at chat GPT answering you super"
+    },
+    {
+      "start": 5104.52,
+      "duration": 0.0,
+      "text": "annoyed at chat GPT answering you super long<01:25:04.719><c> sentences</c><01:25:05.400><c> this</c><01:25:05.520><c> is</c><01:25:05.679><c> because</c><01:25:05.840><c> of</c><01:25:06.000><c> all</c>"
+    },
+    {
+      "start": 5107.149,
+      "duration": 0.0,
+      "text": "long sentences this is because of all"
+    },
+    {
+      "start": 5107.159,
+      "duration": 0.0,
+      "text": "long sentences this is because of all rhf<01:25:08.159><c> um</c><01:25:08.639><c> annotator</c><01:25:09.239><c> distribution</c><01:25:09.719><c> shift</c><01:25:10.600><c> uh</c>"
+    },
+    {
+      "start": 5111.59,
+      "duration": 0.0,
+      "text": "rhf um annotator distribution shift uh"
+    },
+    {
+      "start": 5111.6,
+      "duration": 0.0,
+      "text": "rhf um annotator distribution shift uh like<01:25:11.760><c> the</c><01:25:11.880><c> distribution</c><01:25:12.360><c> of</c><01:25:12.480><c> annotators</c><01:25:13.000><c> that</c>"
+    },
+    {
+      "start": 5113.109,
+      "duration": 0.0,
+      "text": "like the distribution of annotators that"
+    },
+    {
+      "start": 5113.119,
+      "duration": 0.0,
+      "text": "like the distribution of annotators that you<01:25:13.239><c> use</c><01:25:13.800><c> matters</c><01:25:14.159><c> a</c><01:25:14.360><c> lot</c><01:25:14.960><c> and</c><01:25:15.040><c> you</c><01:25:15.199><c> have</c><01:25:15.320><c> to</c>"
+    },
+    {
+      "start": 5115.47,
+      "duration": 0.0,
+      "text": "you use matters a lot and you have to"
+    },
+    {
+      "start": 5115.48,
+      "duration": 0.0,
+      "text": "you use matters a lot and you have to think<01:25:15.800><c> like</c><01:25:16.080><c> what</c><01:25:16.239><c> is</c><01:25:16.719><c> what</c><01:25:16.840><c> is</c><01:25:17.000><c> even</c><01:25:17.159><c> the</c>"
+    },
+    {
+      "start": 5117.31,
+      "duration": 0.0,
+      "text": "think like what is what is even the"
+    },
+    {
+      "start": 5117.32,
+      "duration": 0.0,
+      "text": "think like what is what is even the humans<01:25:17.600><c> that</c><01:25:17.719><c> we</c><01:25:17.800><c> want</c><01:25:17.920><c> to</c><01:25:18.040><c> represent</c><01:25:18.679><c> in</c>"
+    },
+    {
+      "start": 5118.79,
+      "duration": 0.0,
+      "text": "humans that we want to represent in"
+    },
+    {
+      "start": 5118.8,
+      "duration": 0.0,
+      "text": "humans that we want to represent in these<01:25:19.000><c> models</c><01:25:20.000><c> uh</c><01:25:20.119><c> now</c><01:25:20.320><c> the</c><01:25:20.440><c> question</c><01:25:20.679><c> is</c><01:25:20.800><c> like</c>"
+    },
+    {
+      "start": 5120.95,
+      "duration": 0.0,
+      "text": "these models uh now the question is like"
+    },
+    {
+      "start": 5120.96,
+      "duration": 0.0,
+      "text": "these models uh now the question is like crowdsourcing<01:25:21.600><c> ethics</c><01:25:22.520><c> uh</c><01:25:23.119><c> like</c><01:25:23.480><c> usually</c>"
+    },
+    {
+      "start": 5123.95,
+      "duration": 0.0,
+      "text": "crowdsourcing ethics uh like usually"
+    },
+    {
+      "start": 5123.96,
+      "duration": 0.0,
+      "text": "crowdsourcing ethics uh like usually these<01:25:24.560><c> basically</c><01:25:24.880><c> a</c><01:25:25.000><c> lot</c><01:25:25.119><c> of</c><01:25:25.320><c> the</c><01:25:25.480><c> the</c>"
+    },
+    {
+      "start": 5125.59,
+      "duration": 0.0,
+      "text": "these basically a lot of the the"
+    },
+    {
+      "start": 5125.6,
+      "duration": 0.0,
+      "text": "these basically a lot of the the labeling<01:25:26.000><c> that</c><01:25:26.119><c> is</c><01:25:26.280><c> done</c><01:25:27.080><c> um</c><01:25:28.000><c> like</c><01:25:28.440><c> the</c><01:25:28.600><c> people</c>"
+    },
+    {
+      "start": 5128.87,
+      "duration": 0.0,
+      "text": "labeling that is done um like the people"
+    },
+    {
+      "start": 5128.88,
+      "duration": 0.0,
+      "text": "labeling that is done um like the people who<01:25:29.040><c> do</c><01:25:29.320><c> them</c><01:25:29.520><c> are</c><01:25:29.679><c> not</c><01:25:29.960><c> paid</c><01:25:30.239><c> well</c><01:25:30.600><c> and</c><01:25:30.719><c> they</c>"
+    },
+    {
+      "start": 5130.83,
+      "duration": 0.0,
+      "text": "who do them are not paid well and they"
+    },
+    {
+      "start": 5130.84,
+      "duration": 0.0,
+      "text": "who do them are not paid well and they have<01:25:30.960><c> to</c><01:25:31.119><c> go</c><01:25:31.239><c> through</c><01:25:31.400><c> a</c><01:25:31.520><c> lot</c><01:25:31.639><c> of</c><01:25:31.760><c> toxic</c><01:25:32.080><c> data</c>"
+    },
+    {
+      "start": 5132.95,
+      "duration": 0.0,
+      "text": "have to go through a lot of toxic data"
+    },
+    {
+      "start": 5132.96,
+      "duration": 0.0,
+      "text": "have to go through a lot of toxic data uh<01:25:33.159><c> because</c><01:25:33.360><c> you</c><01:25:33.520><c> basically</c><01:25:33.880><c> want</c><01:25:34.119><c> the</c><01:25:34.239><c> model</c>"
+    },
+    {
+      "start": 5134.51,
+      "duration": 0.0,
+      "text": "uh because you basically want the model"
+    },
+    {
+      "start": 5134.52,
+      "duration": 0.0,
+      "text": "uh because you basically want the model to<01:25:34.719><c> avoid</c><01:25:34.960><c> saying</c><01:25:35.320><c> the</c><01:25:35.480><c> toxic</c><01:25:35.840><c> data</c><01:25:36.719><c> um</c><01:25:37.080><c> so</c>"
+    },
+    {
+      "start": 5137.59,
+      "duration": 0.0,
+      "text": "to avoid saying the toxic data um so"
+    },
+    {
+      "start": 5137.6,
+      "duration": 0.0,
+      "text": "to avoid saying the toxic data um so crowdsourcing<01:25:38.320><c> ethics</c>"
+    },
+    {
+      "start": 5139.35,
+      "duration": 0.0,
+      "text": "crowdsourcing ethics"
+    },
+    {
+      "start": 5139.36,
+      "duration": 0.0,
+      "text": "crowdsourcing ethics too<01:25:40.360><c> so</c><01:25:40.679><c> many</c><01:25:41.000><c> challenges</c><01:25:41.440><c> with</c><01:25:41.560><c> human</c><01:25:41.840><c> data</c>"
+    },
+    {
+      "start": 5142.75,
+      "duration": 0.0,
+      "text": "too so many challenges with human data"
+    },
+    {
+      "start": 5142.76,
+      "duration": 0.0,
+      "text": "too so many challenges with human data um<01:25:43.280><c> so</c><01:25:43.600><c> what</c><01:25:43.760><c> we</c><01:25:43.960><c> did</c><01:25:44.360><c> also</c><01:25:44.719><c> last</c><01:25:44.920><c> year</c><01:25:45.400><c> is</c>"
+    },
+    {
+      "start": 5145.75,
+      "duration": 0.0,
+      "text": "um so what we did also last year is"
+    },
+    {
+      "start": 5145.76,
+      "duration": 0.0,
+      "text": "um so what we did also last year is again<01:25:45.960><c> the</c><01:25:46.080><c> same</c><01:25:46.280><c> thing</c><01:25:46.400><c> as</c><01:25:46.560><c> alpaca</c><01:25:47.159><c> just</c><01:25:47.320><c> the</c>"
+    },
+    {
+      "start": 5147.47,
+      "duration": 0.0,
+      "text": "again the same thing as alpaca just the"
+    },
+    {
+      "start": 5147.48,
+      "duration": 0.0,
+      "text": "again the same thing as alpaca just the idea<01:25:47.760><c> of</c><01:25:47.960><c> like</c><01:25:48.119><c> oh</c><01:25:48.320><c> well</c><01:25:48.560><c> they're</c><01:25:48.760><c> challenges</c>"
+    },
+    {
+      "start": 5149.109,
+      "duration": 0.0,
+      "text": "idea of like oh well they're challenges"
+    },
+    {
+      "start": 5149.119,
+      "duration": 0.0,
+      "text": "idea of like oh well they're challenges with<01:25:49.239><c> humans</c><01:25:49.520><c> maybe</c><01:25:49.679><c> we</c><01:25:49.760><c> can</c><01:25:49.880><c> just</c><01:25:50.000><c> replace</c>"
+    },
+    {
+      "start": 5150.31,
+      "duration": 0.0,
+      "text": "with humans maybe we can just replace"
+    },
+    {
+      "start": 5150.32,
+      "duration": 0.0,
+      "text": "with humans maybe we can just replace them<01:25:50.480><c> with</c><01:25:50.880><c> llms</c><01:25:51.880><c> uh</c><01:25:52.080><c> so</c><01:25:52.320><c> what</c><01:25:52.440><c> we</c><01:25:52.600><c> did</c><01:25:52.800><c> is</c>"
+    },
+    {
+      "start": 5153.03,
+      "duration": 0.0,
+      "text": "them with llms uh so what we did is"
+    },
+    {
+      "start": 5153.04,
+      "duration": 0.0,
+      "text": "them with llms uh so what we did is simply<01:25:53.440><c> replace</c>"
+    },
+    {
+      "start": 5154.83,
+      "duration": 0.0,
+      "text": "simply replace"
+    },
+    {
+      "start": 5154.84,
+      "duration": 0.0,
+      "text": "simply replace um<01:25:55.480><c> oh</c><01:25:55.679><c> I</c><01:25:55.840><c> see</c><01:25:56.159><c> that</c><01:25:56.880><c> I'm</c><01:25:57.000><c> just</c><01:25:57.159><c> realizing</c><01:25:57.560><c> that</c>"
+    },
+    {
+      "start": 5157.669,
+      "duration": 0.0,
+      "text": "um oh I see that I'm just realizing that"
+    },
+    {
+      "start": 5157.679,
+      "duration": 0.0,
+      "text": "um oh I see that I'm just realizing that the<01:25:57.840><c> slides</c><01:25:58.080><c> are</c><01:25:58.199><c> not</c><01:25:58.360><c> sented</c><01:25:58.920><c> anyways</c><01:25:59.679><c> uh</c><01:25:59.800><c> you</c>"
+    },
+    {
+      "start": 5159.99,
+      "duration": 0.0,
+      "text": "the slides are not sented anyways uh you"
+    },
+    {
+      "start": 5160.0,
+      "duration": 0.0,
+      "text": "the slides are not sented anyways uh you replace<01:26:00.520><c> a</c><01:26:00.639><c> human</c><01:26:00.920><c> preference</c><01:26:01.320><c> with</c><01:26:01.440><c> LM</c>"
+    },
+    {
+      "start": 5161.79,
+      "duration": 0.0,
+      "text": "replace a human preference with LM"
+    },
+    {
+      "start": 5161.8,
+      "duration": 0.0,
+      "text": "replace a human preference with LM preferences<01:26:02.760><c> uh</c><01:26:02.880><c> so</c><01:26:03.199><c> here</c><01:26:03.560><c> on</c><01:26:03.840><c> this</c><01:26:04.360><c> uh</c><01:26:04.520><c> figure</c>"
+    },
+    {
+      "start": 5164.83,
+      "duration": 0.0,
+      "text": "preferences uh so here on this uh figure"
+    },
+    {
+      "start": 5164.84,
+      "duration": 0.0,
+      "text": "preferences uh so here on this uh figure you<01:26:04.920><c> see</c><01:26:05.080><c> on</c><01:26:05.199><c> the</c><01:26:05.320><c> xaxis</c><01:26:05.880><c> the</c><01:26:06.040><c> price</c><01:26:06.600><c> that</c><01:26:06.760><c> we</c>"
+    },
+    {
+      "start": 5166.95,
+      "duration": 0.0,
+      "text": "you see on the xaxis the price that we"
+    },
+    {
+      "start": 5166.96,
+      "duration": 0.0,
+      "text": "you see on the xaxis the price that we paid<01:26:07.880><c> uh</c><01:26:07.960><c> for</c><01:26:08.199><c> collecting</c><01:26:08.639><c> human</c><01:26:08.920><c> data</c><01:26:09.239><c> it's</c>"
+    },
+    {
+      "start": 5169.43,
+      "duration": 0.0,
+      "text": "paid uh for collecting human data it's"
+    },
+    {
+      "start": 5169.44,
+      "duration": 0.0,
+      "text": "paid uh for collecting human data it's around"
+    },
+    {
+      "start": 5170.31,
+      "duration": 0.0,
+      "text": "around"
+    },
+    {
+      "start": 5170.32,
+      "duration": 0.0,
+      "text": "around $300<01:26:11.320><c> for</c><01:26:11.560><c> 1,000</c><01:26:12.080><c> examples</c><01:26:12.600><c> and</c><01:26:12.719><c> this</c><01:26:12.840><c> is</c><01:26:13.000><c> on</c>"
+    },
+    {
+      "start": 5173.27,
+      "duration": 0.0,
+      "text": "$300 for 1,000 examples and this is on"
+    },
+    {
+      "start": 5173.28,
+      "duration": 0.0,
+      "text": "$300 for 1,000 examples and this is on mechanical<01:26:13.719><c> turkers</c><01:26:14.239><c> which</c><01:26:14.360><c> are</c><01:26:15.159><c> usually</c>"
+    },
+    {
+      "start": 5175.55,
+      "duration": 0.0,
+      "text": "mechanical turkers which are usually"
+    },
+    {
+      "start": 5175.56,
+      "duration": 0.0,
+      "text": "mechanical turkers which are usually like<01:26:15.760><c> cheaper</c><01:26:16.239><c> than</c><01:26:16.560><c> than</c><01:26:16.880><c> maybe</c><01:26:17.119><c> some</c><01:26:17.239><c> of</c><01:26:17.360><c> the</c>"
+    },
+    {
+      "start": 5177.91,
+      "duration": 0.0,
+      "text": "like cheaper than than maybe some of the"
+    },
+    {
+      "start": 5177.92,
+      "duration": 0.0,
+      "text": "like cheaper than than maybe some of the other<01:26:18.920><c> um</c><01:26:19.320><c> companies</c><01:26:19.679><c> that</c><01:26:19.840><c> you</c><01:26:19.920><c> could</c><01:26:20.040><c> go</c>"
+    },
+    {
+      "start": 5180.229,
+      "duration": 0.0,
+      "text": "other um companies that you could go"
+    },
+    {
+      "start": 5180.239,
+      "duration": 0.0,
+      "text": "other um companies that you could go through<01:26:20.760><c> and</c><01:26:20.920><c> on</c><01:26:21.040><c> the</c><01:26:21.480><c> Y</c><01:26:21.719><c> AIS</c><01:26:22.280><c> it's</c><01:26:22.480><c> basically</c>"
+    },
+    {
+      "start": 5182.83,
+      "duration": 0.0,
+      "text": "through and on the Y AIS it's basically"
+    },
+    {
+      "start": 5182.84,
+      "duration": 0.0,
+      "text": "through and on the Y AIS it's basically the<01:26:23.000><c> agreement</c><01:26:23.920><c> with</c><01:26:24.520><c> uh</c><01:26:24.679><c> other</c><01:26:24.920><c> humans</c><01:26:25.360><c> with</c>"
+    },
+    {
+      "start": 5185.51,
+      "duration": 0.0,
+      "text": "the agreement with uh other humans with"
+    },
+    {
+      "start": 5185.52,
+      "duration": 0.0,
+      "text": "the agreement with uh other humans with the<01:26:25.639><c> mode</c><01:26:25.920><c> of</c><01:26:26.119><c> other</c><01:26:26.360><c> humans</c><01:26:27.119><c> and</c><01:26:27.280><c> what</c><01:26:27.400><c> you</c>"
+    },
+    {
+      "start": 5187.47,
+      "duration": 0.0,
+      "text": "the mode of other humans and what you"
+    },
+    {
+      "start": 5187.48,
+      "duration": 0.0,
+      "text": "the mode of other humans and what you see<01:26:27.679><c> is</c><01:26:27.800><c> that</c><01:26:28.000><c> actually</c><01:26:28.400><c> as</c><01:26:28.520><c> I</c><01:26:28.600><c> told</c><01:26:28.800><c> you</c>"
+    },
+    {
+      "start": 5188.95,
+      "duration": 0.0,
+      "text": "see is that actually as I told you"
+    },
+    {
+      "start": 5188.96,
+      "duration": 0.0,
+      "text": "see is that actually as I told you before<01:26:29.199><c> labeling</c><01:26:29.600><c> is</c><01:26:29.760><c> really</c><01:26:29.960><c> complicated</c>"
+    },
+    {
+      "start": 5190.75,
+      "duration": 0.0,
+      "text": "before labeling is really complicated"
+    },
+    {
+      "start": 5190.76,
+      "duration": 0.0,
+      "text": "before labeling is really complicated humans<01:26:31.159><c> agree</c><01:26:31.560><c> with</c><01:26:31.920><c> themselves</c><01:26:32.639><c> only</c><01:26:32.960><c> around</c>"
+    },
+    {
+      "start": 5193.229,
+      "duration": 0.0,
+      "text": "humans agree with themselves only around"
+    },
+    {
+      "start": 5193.239,
+      "duration": 0.0,
+      "text": "humans agree with themselves only around 66%<01:26:34.000><c> of</c><01:26:34.119><c> the</c><01:26:34.239><c> time</c><01:26:34.880><c> on</c><01:26:35.000><c> a</c><01:26:35.119><c> binary</c><01:26:35.520><c> Tas</c><01:26:36.199><c> and</c><01:26:36.400><c> it's</c>"
+    },
+    {
+      "start": 5196.629,
+      "duration": 0.0,
+      "text": "66% of the time on a binary Tas and it's"
+    },
+    {
+      "start": 5196.639,
+      "duration": 0.0,
+      "text": "66% of the time on a binary Tas and it's not<01:26:36.880><c> that</c><01:26:37.000><c> the</c><01:26:37.239><c> humans</c><01:26:37.520><c> are</c><01:26:37.679><c> not</c><01:26:37.880><c> good</c><01:26:38.159><c> here</c>"
+    },
+    {
+      "start": 5198.669,
+      "duration": 0.0,
+      "text": "not that the humans are not good here"
+    },
+    {
+      "start": 5198.679,
+      "duration": 0.0,
+      "text": "not that the humans are not good here because<01:26:39.239><c> uh</c><01:26:39.360><c> we</c><01:26:39.440><c> were</c><01:26:39.760><c> five</c><01:26:40.040><c> main</c><01:26:40.320><c> authors</c><01:26:40.639><c> on</c>"
+    },
+    {
+      "start": 5200.75,
+      "duration": 0.0,
+      "text": "because uh we were five main authors on"
+    },
+    {
+      "start": 5200.76,
+      "duration": 0.0,
+      "text": "because uh we were five main authors on this<01:26:40.960><c> paper</c><01:26:41.480><c> we</c><01:26:41.639><c> tried</c><01:26:41.920><c> to</c><01:26:42.080><c> label</c><01:26:42.960><c> this</c><01:26:43.159><c> data</c>"
+    },
+    {
+      "start": 5203.43,
+      "duration": 0.0,
+      "text": "this paper we tried to label this data"
+    },
+    {
+      "start": 5203.44,
+      "duration": 0.0,
+      "text": "this paper we tried to label this data ourselves<01:26:44.159><c> and</c><01:26:44.320><c> we</c><01:26:44.480><c> only</c><01:26:44.760><c> had</c><01:26:44.960><c> like</c><01:26:45.199><c> say</c><01:26:45.480><c> 67</c><01:26:46.080><c> or</c>"
+    },
+    {
+      "start": 5206.229,
+      "duration": 0.0,
+      "text": "ourselves and we only had like say 67 or"
+    },
+    {
+      "start": 5206.239,
+      "duration": 0.0,
+      "text": "ourselves and we only had like say 67 or 68%<01:26:47.040><c> accuracy</c><01:26:47.719><c> even</c><01:26:47.960><c> though</c><01:26:48.119><c> we</c><01:26:48.400><c> talk</c><01:26:48.679><c> like</c><01:26:48.760><c> we</c>"
+    },
+    {
+      "start": 5208.87,
+      "duration": 0.0,
+      "text": "68% accuracy even though we talk like we"
+    },
+    {
+      "start": 5208.88,
+      "duration": 0.0,
+      "text": "68% accuracy even though we talk like we talk<01:26:49.080><c> for</c><01:26:49.199><c> like</c><01:26:49.360><c> 3</c><01:26:49.560><c> hours</c><01:26:49.840><c> of</c><01:26:50.000><c> how</c><01:26:50.119><c> we</c><01:26:50.199><c> should</c>"
+    },
+    {
+      "start": 5210.35,
+      "duration": 0.0,
+      "text": "talk for like 3 hours of how we should"
+    },
+    {
+      "start": 5210.36,
+      "duration": 0.0,
+      "text": "talk for like 3 hours of how we should be<01:26:50.520><c> doing</c><01:26:50.760><c> labeling</c><01:26:51.600><c> really</c><01:26:51.760><c> it's</c>"
+    },
+    {
+      "start": 5211.99,
+      "duration": 0.0,
+      "text": "be doing labeling really it's"
+    },
+    {
+      "start": 5212.0,
+      "duration": 0.0,
+      "text": "be doing labeling really it's complicated<01:26:52.679><c> it's</c><01:26:52.840><c> not</c><01:26:53.000><c> an</c><01:26:53.159><c> easy</c><01:26:53.440><c> task</c><01:26:54.119><c> um</c><01:26:54.440><c> and</c>"
+    },
+    {
+      "start": 5214.59,
+      "duration": 0.0,
+      "text": "complicated it's not an easy task um and"
+    },
+    {
+      "start": 5214.6,
+      "duration": 0.0,
+      "text": "complicated it's not an easy task um and here<01:26:54.760><c> I</c><01:26:54.880><c> just</c><01:26:55.000><c> showed</c><01:26:55.280><c> many</c><01:26:55.480><c> different</c><01:26:55.719><c> models</c>"
+    },
+    {
+      "start": 5216.47,
+      "duration": 0.0,
+      "text": "here I just showed many different models"
+    },
+    {
+      "start": 5216.48,
+      "duration": 0.0,
+      "text": "here I just showed many different models and<01:26:56.880><c> um</c><01:26:57.280><c> basically</c><01:26:57.639><c> you</c><01:26:57.760><c> see</c><01:26:57.960><c> that</c><01:26:58.119><c> models</c><01:26:58.400><c> are</c>"
+    },
+    {
+      "start": 5218.55,
+      "duration": 0.0,
+      "text": "and um basically you see that models are"
+    },
+    {
+      "start": 5218.56,
+      "duration": 0.0,
+      "text": "and um basically you see that models are much<01:26:58.760><c> cheaper</c><01:26:59.400><c> and</c><01:26:59.520><c> they</c><01:26:59.679><c> can</c><01:26:59.840><c> actually</c><01:27:00.119><c> get</c>"
+    },
+    {
+      "start": 5220.43,
+      "duration": 0.0,
+      "text": "much cheaper and they can actually get"
+    },
+    {
+      "start": 5220.44,
+      "duration": 0.0,
+      "text": "much cheaper and they can actually get higher<01:27:01.080><c> agreement</c><01:27:01.560><c> with</c><01:27:01.880><c> the</c><01:27:01.960><c> mode</c><01:27:02.239><c> of</c><01:27:02.360><c> humans</c>"
+    },
+    {
+      "start": 5222.87,
+      "duration": 0.0,
+      "text": "higher agreement with the mode of humans"
+    },
+    {
+      "start": 5222.88,
+      "duration": 0.0,
+      "text": "higher agreement with the mode of humans than<01:27:03.119><c> human</c><01:27:03.600><c> humans</c><01:27:04.080><c> themselves</c><01:27:04.679><c> and</c><01:27:04.800><c> the</c>"
+    },
+    {
+      "start": 5224.91,
+      "duration": 0.0,
+      "text": "than human humans themselves and the"
+    },
+    {
+      "start": 5224.92,
+      "duration": 0.0,
+      "text": "than human humans themselves and the reason<01:27:05.199><c> why</c><01:27:05.360><c> is</c><01:27:05.480><c> because</c><01:27:05.639><c> humans</c><01:27:05.920><c> have</c><01:27:06.040><c> a</c><01:27:06.119><c> lot</c>"
+    },
+    {
+      "start": 5226.229,
+      "duration": 0.0,
+      "text": "reason why is because humans have a lot"
+    },
+    {
+      "start": 5226.239,
+      "duration": 0.0,
+      "text": "reason why is because humans have a lot of<01:27:06.320><c> varant</c><01:27:06.880><c> models</c><01:27:07.239><c> have</c><01:27:07.360><c> no</c><01:27:07.520><c> varant</c><01:27:07.920><c> so</c><01:27:08.080><c> they</c>"
+    },
+    {
+      "start": 5228.149,
+      "duration": 0.0,
+      "text": "of varant models have no varant so they"
+    },
+    {
+      "start": 5228.159,
+      "duration": 0.0,
+      "text": "of varant models have no varant so they might<01:27:08.320><c> be</c><01:27:08.400><c> a</c><01:27:08.480><c> little</c><01:27:08.600><c> bit</c><01:27:08.760><c> more</c><01:27:08.920><c> biased</c><01:27:09.560><c> but</c>"
+    },
+    {
+      "start": 5229.709,
+      "duration": 0.0,
+      "text": "might be a little bit more biased but"
+    },
+    {
+      "start": 5229.719,
+      "duration": 0.0,
+      "text": "might be a little bit more biased but have<01:27:09.920><c> less</c><01:27:10.280><c> virence</c><01:27:11.280><c> uh</c><01:27:11.639><c> so</c><01:27:11.800><c> it</c><01:27:11.920><c> works</c>"
+    },
+    {
+      "start": 5232.149,
+      "duration": 0.0,
+      "text": "have less virence uh so it works"
+    },
+    {
+      "start": 5232.159,
+      "duration": 0.0,
+      "text": "have less virence uh so it works surprisingly<01:27:12.760><c> well</c><01:27:13.440><c> and</c><01:27:13.639><c> now</c><01:27:13.840><c> it's</c><01:27:14.040><c> kind</c><01:27:14.159><c> of</c>"
+    },
+    {
+      "start": 5234.31,
+      "duration": 0.0,
+      "text": "surprisingly well and now it's kind of"
+    },
+    {
+      "start": 5234.32,
+      "duration": 0.0,
+      "text": "surprisingly well and now it's kind of the<01:27:14.480><c> standard</c><01:27:14.840><c> in</c><01:27:15.119><c> open</c><01:27:15.679><c> uh</c><01:27:15.840><c> Source</c><01:27:16.159><c> Community</c>"
+    },
+    {
+      "start": 5236.709,
+      "duration": 0.0,
+      "text": "the standard in open uh Source Community"
+    },
+    {
+      "start": 5236.719,
+      "duration": 0.0,
+      "text": "the standard in open uh Source Community I<01:27:16.800><c> think</c><01:27:16.960><c> even</c><01:27:17.119><c> in</c><01:27:17.400><c> Industry</c><01:27:18.199><c> a</c><01:27:18.360><c> lot</c><01:27:18.480><c> of</c><01:27:18.639><c> people</c>"
+    },
+    {
+      "start": 5238.87,
+      "duration": 0.0,
+      "text": "I think even in Industry a lot of people"
+    },
+    {
+      "start": 5238.88,
+      "duration": 0.0,
+      "text": "I think even in Industry a lot of people use<01:27:19.199><c> both</c><01:27:19.440><c> humans</c><01:27:19.840><c> and</c><01:27:20.000><c> llms</c><01:27:20.760><c> for</c><01:27:21.000><c> improving</c>"
+    },
+    {
+      "start": 5241.709,
+      "duration": 0.0,
+      "text": "use both humans and llms for improving"
+    },
+    {
+      "start": 5241.719,
+      "duration": 0.0,
+      "text": "use both humans and llms for improving uh<01:27:21.840><c> the</c><01:27:21.960><c> colle</c><01:27:22.440><c> collection</c><01:27:22.760><c> of</c><01:27:22.880><c> allf</c><01:27:23.400><c> data</c>"
+    },
+    {
+      "start": 5244.629,
+      "duration": 0.0,
+      "text": "uh the colle collection of allf data"
+    },
+    {
+      "start": 5244.639,
+      "duration": 0.0,
+      "text": "uh the colle collection of allf data um<01:27:25.159><c> and</c><01:27:25.360><c> this</c><01:27:25.480><c> is</c><01:27:25.679><c> like</c><01:27:25.880><c> this</c><01:27:25.960><c> is</c><01:27:26.159><c> the</c><01:27:26.280><c> paper</c>"
+    },
+    {
+      "start": 5246.51,
+      "duration": 0.0,
+      "text": "um and this is like this is the paper"
+    },
+    {
+      "start": 5246.52,
+      "duration": 0.0,
+      "text": "um and this is like this is the paper from<01:27:26.760><c> last</c><01:27:26.960><c> year</c><01:27:27.199><c> but</c><01:27:27.400><c> honestly</c><01:27:27.840><c> now</c><01:27:28.000><c> it's</c>"
+    },
+    {
+      "start": 5248.149,
+      "duration": 0.0,
+      "text": "from last year but honestly now it's"
+    },
+    {
+      "start": 5248.159,
+      "duration": 0.0,
+      "text": "from last year but honestly now it's more<01:27:28.400><c> like</c><01:27:29.280><c> that</c><01:27:29.480><c> llms</c><01:27:29.920><c> would</c><01:27:30.080><c> be</c><01:27:30.239><c> around</c><01:27:30.560><c> this</c>"
+    },
+    {
+      "start": 5250.95,
+      "duration": 0.0,
+      "text": "more like that llms would be around this"
+    },
+    {
+      "start": 5250.96,
+      "duration": 0.0,
+      "text": "more like that llms would be around this agreement<01:27:31.320><c> and</c><01:27:31.480><c> this</c><01:27:31.639><c> cost</c><01:27:31.880><c> so</c><01:27:32.000><c> around</c><01:27:32.560><c> I</c>"
+    },
+    {
+      "start": 5252.629,
+      "duration": 0.0,
+      "text": "agreement and this cost so around I"
+    },
+    {
+      "start": 5252.639,
+      "duration": 0.0,
+      "text": "agreement and this cost so around I would<01:27:32.800><c> say</c><01:27:32.960><c> 50x</c><01:27:33.520><c> cheaper</c><01:27:33.880><c> than</c><01:27:34.080><c> humans</c><01:27:34.719><c> and</c>"
+    },
+    {
+      "start": 5254.87,
+      "duration": 0.0,
+      "text": "would say 50x cheaper than humans and"
+    },
+    {
+      "start": 5254.88,
+      "duration": 0.0,
+      "text": "would say 50x cheaper than humans and better<01:27:35.159><c> agreement</c><01:27:35.560><c> with</c><01:27:35.719><c> human</c><01:27:36.440><c> than</c><01:27:36.679><c> humans</c>"
+    },
+    {
+      "start": 5258.189,
+      "duration": 0.0,
+      "text": "better agreement with human than humans"
+    },
+    {
+      "start": 5258.199,
+      "duration": 0.0,
+      "text": "better agreement with human than humans themselves<01:27:39.199><c> okay</c><01:27:39.960><c> so</c><01:27:40.960><c> that</c><01:27:41.199><c> gets</c><01:27:41.440><c> us</c><01:27:41.600><c> to</c>"
+    },
+    {
+      "start": 5261.83,
+      "duration": 0.0,
+      "text": "themselves okay so that gets us to"
+    },
+    {
+      "start": 5261.84,
+      "duration": 0.0,
+      "text": "themselves okay so that gets us to evaluation<01:27:42.320><c> of</c><01:27:42.520><c> post</c>"
+    },
+    {
+      "start": 5263.47,
+      "duration": 0.0,
+      "text": "evaluation of post"
+    },
+    {
+      "start": 5263.48,
+      "duration": 0.0,
+      "text": "evaluation of post training<01:27:44.480><c> um</c><01:27:45.199><c> that</c><01:27:45.360><c> goes</c><01:27:45.600><c> back</c><01:27:45.760><c> to</c><01:27:45.920><c> your</c>"
+    },
+    {
+      "start": 5266.109,
+      "duration": 0.0,
+      "text": "training um that goes back to your"
+    },
+    {
+      "start": 5266.119,
+      "duration": 0.0,
+      "text": "training um that goes back to your initial<01:27:46.520><c> question</c><01:27:46.760><c> at</c><01:27:46.880><c> the</c><01:27:46.960><c> beginning</c><01:27:47.199><c> of</c><01:27:47.280><c> the</c>"
+    },
+    {
+      "start": 5267.39,
+      "duration": 0.0,
+      "text": "initial question at the beginning of the"
+    },
+    {
+      "start": 5267.4,
+      "duration": 0.0,
+      "text": "initial question at the beginning of the lecture<01:27:47.960><c> how</c><01:27:48.159><c> do</c><01:27:48.239><c> you</c><01:27:48.400><c> evaluate</c><01:27:48.760><c> something</c>"
+    },
+    {
+      "start": 5268.99,
+      "duration": 0.0,
+      "text": "lecture how do you evaluate something"
+    },
+    {
+      "start": 5269.0,
+      "duration": 0.0,
+      "text": "lecture how do you evaluate something like<01:27:49.280><c> chpt</c><01:27:50.280><c> uh</c><01:27:50.400><c> the</c><01:27:50.600><c> answers</c><01:27:50.880><c> that</c><01:27:51.000><c> chpt</c><01:27:51.560><c> could</c>"
+    },
+    {
+      "start": 5271.75,
+      "duration": 0.0,
+      "text": "like chpt uh the answers that chpt could"
+    },
+    {
+      "start": 5271.76,
+      "duration": 0.0,
+      "text": "like chpt uh the answers that chpt could give<01:27:52.560><c> are</c><01:27:52.880><c> basically</c><01:27:53.360><c> unbounded</c><01:27:54.560><c> and</c><01:27:54.719><c> it's</c>"
+    },
+    {
+      "start": 5274.87,
+      "duration": 0.0,
+      "text": "give are basically unbounded and it's"
+    },
+    {
+      "start": 5274.88,
+      "duration": 0.0,
+      "text": "give are basically unbounded and it's not<01:27:55.080><c> that</c><01:27:55.199><c> there</c><01:27:55.440><c> one</c><01:27:55.679><c> right</c><01:27:55.920><c> answer</c><01:27:56.360><c> there</c>"
+    },
+    {
+      "start": 5276.43,
+      "duration": 0.0,
+      "text": "not that there one right answer there"
+    },
+    {
+      "start": 5276.44,
+      "duration": 0.0,
+      "text": "not that there one right answer there are<01:27:56.639><c> many</c><01:27:56.920><c> answers</c><01:27:57.320><c> that</c><01:27:57.440><c> are</c><01:27:57.639><c> just</c><01:27:57.800><c> as</c><01:27:58.040><c> good</c>"
+    },
+    {
+      "start": 5278.87,
+      "duration": 0.0,
+      "text": "are many answers that are just as good"
+    },
+    {
+      "start": 5278.88,
+      "duration": 0.0,
+      "text": "are many answers that are just as good um<01:27:59.280><c> so</c><01:27:59.440><c> there</c><01:27:59.520><c> are</c><01:27:59.639><c> many</c><01:27:59.840><c> challenges</c><01:28:00.560><c> one</c><01:28:01.119><c> you</c>"
+    },
+    {
+      "start": 5281.35,
+      "duration": 0.0,
+      "text": "um so there are many challenges one you"
+    },
+    {
+      "start": 5281.36,
+      "duration": 0.0,
+      "text": "um so there are many challenges one you can't<01:28:01.719><c> use</c><01:28:02.400><c> validation</c><01:28:02.840><c> loss</c><01:28:03.639><c> because</c><01:28:04.600><c> one</c>"
+    },
+    {
+      "start": 5284.99,
+      "duration": 0.0,
+      "text": "can't use validation loss because one"
+    },
+    {
+      "start": 5285.0,
+      "duration": 0.0,
+      "text": "can't use validation loss because one method<01:28:05.280><c> might</c><01:28:05.400><c> use</c><01:28:05.600><c> po</c><01:28:06.000><c> the</c><01:28:06.119><c> other</c><01:28:06.239><c> one</c><01:28:06.400><c> might</c>"
+    },
+    {
+      "start": 5286.51,
+      "duration": 0.0,
+      "text": "method might use po the other one might"
+    },
+    {
+      "start": 5286.52,
+      "duration": 0.0,
+      "text": "method might use po the other one might use<01:28:06.679><c> DPO</c><01:28:07.280><c> validation</c><01:28:07.679><c> loss</c><01:28:07.960><c> is</c><01:28:08.080><c> not</c>"
+    },
+    {
+      "start": 5288.189,
+      "duration": 0.0,
+      "text": "use DPO validation loss is not"
+    },
+    {
+      "start": 5288.199,
+      "duration": 0.0,
+      "text": "use DPO validation loss is not comparable<01:28:09.159><c> second</c><01:28:09.520><c> you</c><01:28:09.679><c> can't</c><01:28:09.840><c> use</c><01:28:10.159><c> Cal</c><01:28:10.639><c> uh</c>"
+    },
+    {
+      "start": 5290.75,
+      "duration": 0.0,
+      "text": "comparable second you can't use Cal uh"
+    },
+    {
+      "start": 5290.76,
+      "duration": 0.0,
+      "text": "comparable second you can't use Cal uh sorry<01:28:11.080><c> perplexity</c><01:28:11.880><c> that's</c><01:28:12.000><c> the</c><01:28:12.159><c> thing</c><01:28:12.280><c> I</c><01:28:12.400><c> told</c>"
+    },
+    {
+      "start": 5292.55,
+      "duration": 0.0,
+      "text": "sorry perplexity that's the thing I told"
+    },
+    {
+      "start": 5292.56,
+      "duration": 0.0,
+      "text": "sorry perplexity that's the thing I told you<01:28:12.719><c> before</c><01:28:13.480><c> these</c><01:28:13.760><c> models</c><01:28:14.760><c> uh</c><01:28:14.920><c> are</c><01:28:15.080><c> not</c>"
+    },
+    {
+      "start": 5295.35,
+      "duration": 0.0,
+      "text": "you before these models uh are not"
+    },
+    {
+      "start": 5295.36,
+      "duration": 0.0,
+      "text": "you before these models uh are not calibrated<01:28:15.920><c> they</c><01:28:16.000><c> don't</c><01:28:16.239><c> give</c><01:28:16.679><c> distributions</c>"
+    },
+    {
+      "start": 5297.669,
+      "duration": 0.0,
+      "text": "calibrated they don't give distributions"
+    },
+    {
+      "start": 5297.679,
+      "duration": 0.0,
+      "text": "calibrated they don't give distributions they<01:28:17.840><c> they</c><01:28:17.960><c> just</c><01:28:18.239><c> optimize</c><01:28:18.719><c> for</c><01:28:18.920><c> one</c><01:28:19.119><c> thing</c><01:28:19.400><c> so</c>"
+    },
+    {
+      "start": 5299.51,
+      "duration": 0.0,
+      "text": "they they just optimize for one thing so"
+    },
+    {
+      "start": 5299.52,
+      "duration": 0.0,
+      "text": "they they just optimize for one thing so you<01:28:19.639><c> can't</c><01:28:19.840><c> use</c><01:28:20.040><c> perplexity</c><01:28:20.560><c> for</c><01:28:20.800><c> actually</c>"
+    },
+    {
+      "start": 5301.109,
+      "duration": 0.0,
+      "text": "you can't use perplexity for actually"
+    },
+    {
+      "start": 5301.119,
+      "duration": 0.0,
+      "text": "you can't use perplexity for actually evaluating<01:28:22.119><c> uh</c><01:28:22.239><c> these</c><01:28:22.400><c> type</c><01:28:22.600><c> of</c><01:28:22.679><c> models</c><01:28:23.040><c> once</c>"
+    },
+    {
+      "start": 5303.189,
+      "duration": 0.0,
+      "text": "evaluating uh these type of models once"
+    },
+    {
+      "start": 5303.199,
+      "duration": 0.0,
+      "text": "evaluating uh these type of models once they're<01:28:23.400><c> aligned</c><01:28:24.400><c> sorry</c><01:28:24.719><c> one</c><01:28:24.960><c> Z</c><01:28:25.679><c> lined</c><01:28:26.679><c> third</c>"
+    },
+    {
+      "start": 5307.51,
+      "duration": 0.0,
+      "text": "they're aligned sorry one Z lined third"
+    },
+    {
+      "start": 5307.52,
+      "duration": 0.0,
+      "text": "they're aligned sorry one Z lined third uh<01:28:27.639><c> there's</c><01:28:27.760><c> a</c><01:28:27.880><c> large</c><01:28:28.199><c> diversity</c><01:28:28.639><c> of</c>"
+    },
+    {
+      "start": 5308.79,
+      "duration": 0.0,
+      "text": "uh there's a large diversity of"
+    },
+    {
+      "start": 5308.8,
+      "duration": 0.0,
+      "text": "uh there's a large diversity of questions<01:28:29.040><c> that</c><01:28:29.199><c> human</c><01:28:29.520><c> might</c><01:28:29.840><c> ask</c><01:28:30.159><c> to</c><01:28:30.360><c> these</c>"
+    },
+    {
+      "start": 5310.51,
+      "duration": 0.0,
+      "text": "questions that human might ask to these"
+    },
+    {
+      "start": 5310.52,
+      "duration": 0.0,
+      "text": "questions that human might ask to these models<01:28:31.119><c> generation</c><01:28:31.800><c> open</c><01:28:32.159><c> QA</c><01:28:32.719><c> like</c><01:28:32.960><c> some</c>"
+    },
+    {
+      "start": 5313.189,
+      "duration": 0.0,
+      "text": "models generation open QA like some"
+    },
+    {
+      "start": 5313.199,
+      "duration": 0.0,
+      "text": "models generation open QA like some question<01:28:33.480><c> answering</c><01:28:34.239><c> some</c><01:28:34.440><c> summarization</c>"
+    },
+    {
+      "start": 5315.229,
+      "duration": 0.0,
+      "text": "question answering some summarization"
+    },
+    {
+      "start": 5315.239,
+      "duration": 0.0,
+      "text": "question answering some summarization and<01:28:35.360><c> all</c><01:28:35.480><c> of</c><01:28:35.639><c> these</c><01:28:35.760><c> things</c><01:28:35.960><c> so</c><01:28:36.080><c> there's</c><01:28:36.239><c> so</c>"
+    },
+    {
+      "start": 5316.35,
+      "duration": 0.0,
+      "text": "and all of these things so there's so"
+    },
+    {
+      "start": 5316.36,
+      "duration": 0.0,
+      "text": "and all of these things so there's so many<01:28:36.520><c> things</c><01:28:36.679><c> you</c><01:28:36.800><c> have</c><01:28:36.920><c> to</c><01:28:37.360><c> cover</c><01:28:38.360><c> um</c><01:28:39.119><c> then</c>"
+    },
+    {
+      "start": 5319.39,
+      "duration": 0.0,
+      "text": "many things you have to cover um then"
+    },
+    {
+      "start": 5319.4,
+      "duration": 0.0,
+      "text": "many things you have to cover um then the<01:28:39.600><c> tasks</c><01:28:39.880><c> are</c><01:28:40.080><c> really</c><01:28:40.320><c> open-ended</c><01:28:41.080><c> so</c><01:28:41.239><c> it's</c>"
+    },
+    {
+      "start": 5321.43,
+      "duration": 0.0,
+      "text": "the tasks are really open-ended so it's"
+    },
+    {
+      "start": 5321.44,
+      "duration": 0.0,
+      "text": "the tasks are really open-ended so it's very<01:28:41.639><c> hard</c><01:28:41.840><c> to</c><01:28:42.000><c> automate</c><01:28:42.400><c> so</c><01:28:42.600><c> that's</c><01:28:42.880><c> what</c><01:28:43.000><c> you</c>"
+    },
+    {
+      "start": 5323.109,
+      "duration": 0.0,
+      "text": "very hard to automate so that's what you"
+    },
+    {
+      "start": 5323.119,
+      "duration": 0.0,
+      "text": "very hard to automate so that's what you were<01:28:43.400><c> alluding</c><01:28:43.760><c> to</c><01:28:44.440><c> before</c><01:28:45.440><c> so</c><01:28:45.679><c> the</c><01:28:45.840><c> idea</c><01:28:46.760><c> uh</c>"
+    },
+    {
+      "start": 5326.95,
+      "duration": 0.0,
+      "text": "were alluding to before so the idea uh"
+    },
+    {
+      "start": 5326.96,
+      "duration": 0.0,
+      "text": "were alluding to before so the idea uh is<01:28:47.159><c> that</c><01:28:47.360><c> instead</c><01:28:47.679><c> of</c><01:28:47.800><c> trying</c><01:28:48.080><c> to</c><01:28:48.239><c> come</c><01:28:48.400><c> up</c>"
+    },
+    {
+      "start": 5328.59,
+      "duration": 0.0,
+      "text": "is that instead of trying to come up"
+    },
+    {
+      "start": 5328.6,
+      "duration": 0.0,
+      "text": "is that instead of trying to come up with<01:28:49.000><c> really</c><01:28:49.280><c> easily</c><01:28:49.679><c> automated</c><01:28:50.560><c> uh</c>"
+    },
+    {
+      "start": 5330.83,
+      "duration": 0.0,
+      "text": "with really easily automated uh"
+    },
+    {
+      "start": 5330.84,
+      "duration": 0.0,
+      "text": "with really easily automated uh benchmarks<01:28:51.840><c> uh</c><01:28:52.040><c> it's</c><01:28:52.239><c> just</c><01:28:52.400><c> we're</c><01:28:52.520><c> going</c><01:28:52.639><c> to</c>"
+    },
+    {
+      "start": 5332.83,
+      "duration": 0.0,
+      "text": "benchmarks uh it's just we're going to"
+    },
+    {
+      "start": 5332.84,
+      "duration": 0.0,
+      "text": "benchmarks uh it's just we're going to ask<01:28:53.119><c> questions</c><01:28:53.560><c> that</c><01:28:53.880><c> that</c><01:28:54.360><c> users</c><01:28:54.760><c> actually</c>"
+    },
+    {
+      "start": 5334.95,
+      "duration": 0.0,
+      "text": "ask questions that that users actually"
+    },
+    {
+      "start": 5334.96,
+      "duration": 0.0,
+      "text": "ask questions that that users actually ask<01:28:55.199><c> to</c><01:28:55.360><c> these</c><01:28:55.520><c> models</c><01:28:55.920><c> in</c><01:28:56.119><c> practice</c><01:28:56.800><c> and</c>"
+    },
+    {
+      "start": 5336.91,
+      "duration": 0.0,
+      "text": "ask to these models in practice and"
+    },
+    {
+      "start": 5336.92,
+      "duration": 0.0,
+      "text": "ask to these models in practice and we're<01:28:57.080><c> just</c><01:28:57.199><c> going</c><01:28:57.320><c> to</c><01:28:57.520><c> ask</c><01:28:57.760><c> annotators</c><01:28:58.560><c> to</c>"
+    },
+    {
+      "start": 5338.75,
+      "duration": 0.0,
+      "text": "we're just going to ask annotators to"
+    },
+    {
+      "start": 5338.76,
+      "duration": 0.0,
+      "text": "we're just going to ask annotators to say<01:28:59.320><c> between</c><01:28:59.760><c> these</c><01:28:59.920><c> two</c><01:29:00.159><c> models</c><01:29:00.800><c> which</c><01:29:00.920><c> one</c>"
+    },
+    {
+      "start": 5341.07,
+      "duration": 0.0,
+      "text": "say between these two models which one"
+    },
+    {
+      "start": 5341.08,
+      "duration": 0.0,
+      "text": "say between these two models which one is<01:29:01.280><c> better</c><01:29:01.639><c> like</c><01:29:01.760><c> what's</c><01:29:01.960><c> the</c><01:29:02.239><c> what's</c><01:29:02.400><c> the</c>"
+    },
+    {
+      "start": 5342.47,
+      "duration": 0.0,
+      "text": "is better like what's the what's the"
+    },
+    {
+      "start": 5342.48,
+      "duration": 0.0,
+      "text": "is better like what's the what's the better<01:29:02.719><c> output</c><01:29:03.040><c> so</c><01:29:03.239><c> basically</c><01:29:03.600><c> do</c><01:29:03.800><c> exact</c><01:29:04.119><c> same</c>"
+    },
+    {
+      "start": 5344.35,
+      "duration": 0.0,
+      "text": "better output so basically do exact same"
+    },
+    {
+      "start": 5344.36,
+      "duration": 0.0,
+      "text": "better output so basically do exact same thing<01:29:05.239><c> as</c><01:29:06.159><c> um</c><01:29:07.119><c> basically</c><01:29:07.520><c> the</c><01:29:07.679><c> data</c><01:29:07.920><c> from</c><01:29:08.080><c> rhf</c>"
+    },
+    {
+      "start": 5348.629,
+      "duration": 0.0,
+      "text": "thing as um basically the data from rhf"
+    },
+    {
+      "start": 5348.639,
+      "duration": 0.0,
+      "text": "thing as um basically the data from rhf but<01:29:08.719><c> you</c><01:29:08.840><c> use</c><01:29:08.960><c> it</c><01:29:09.080><c> now</c><01:29:09.239><c> for</c><01:29:09.440><c> evaluation</c><01:29:10.320><c> yes</c>"
+    },
+    {
+      "start": 5350.59,
+      "duration": 0.0,
+      "text": "but you use it now for evaluation yes"
+    },
+    {
+      "start": 5350.6,
+      "duration": 0.0,
+      "text": "but you use it now for evaluation yes I'm<01:29:10.719><c> not</c><01:29:10.880><c> sure</c><01:29:11.040><c> I</c><01:29:11.400><c> understand</c><01:29:11.560><c> what</c><01:29:11.639><c> you</c><01:29:11.760><c> mean</c>"
+    },
+    {
+      "start": 5351.99,
+      "duration": 0.0,
+      "text": "I'm not sure I understand what you mean"
+    },
+    {
+      "start": 5352.0,
+      "duration": 0.0,
+      "text": "I'm not sure I understand what you mean by<01:29:12.199><c> like</c><01:29:12.400><c> can't</c><01:29:12.560><c> use</c><01:29:12.760><c> perplexity</c><01:29:13.239><c> and</c><01:29:13.360><c> not</c>"
+    },
+    {
+      "start": 5353.59,
+      "duration": 0.0,
+      "text": "by like can't use perplexity and not"
+    },
+    {
+      "start": 5353.6,
+      "duration": 0.0,
+      "text": "by like can't use perplexity and not calibrated<01:29:14.239><c> right</c><01:29:14.440><c> like</c><01:29:14.920><c> LM</c><01:29:15.360><c> is</c><01:29:15.520><c> still</c><01:29:15.800><c> doing</c>"
+    },
+    {
+      "start": 5356.189,
+      "duration": 0.0,
+      "text": "calibrated right like LM is still doing"
+    },
+    {
+      "start": 5356.199,
+      "duration": 0.0,
+      "text": "calibrated right like LM is still doing like<01:29:16.400><c> next</c><01:29:16.800><c> token</c>"
+    },
+    {
+      "start": 5358.109,
+      "duration": 0.0,
+      "text": "like next token"
+    },
+    {
+      "start": 5358.119,
+      "duration": 0.0,
+      "text": "like next token prediction<01:29:19.119><c> so</c><01:29:19.760><c> I</c><01:29:20.159><c> can't</c><01:29:21.159><c> so</c><01:29:21.400><c> think</c><01:29:21.600><c> about</c><01:29:22.440><c> um</c>"
+    },
+    {
+      "start": 5363.43,
+      "duration": 0.0,
+      "text": "prediction so I can't so think about um"
+    },
+    {
+      "start": 5363.44,
+      "duration": 0.0,
+      "text": "prediction so I can't so think about um the<01:29:23.639><c> optim</c><01:29:24.320><c> solution</c><01:29:24.719><c> after</c><01:29:24.920><c> doing</c><01:29:25.199><c> PO</c><01:29:25.800><c> is</c>"
+    },
+    {
+      "start": 5366.03,
+      "duration": 0.0,
+      "text": "the optim solution after doing PO is"
+    },
+    {
+      "start": 5366.04,
+      "duration": 0.0,
+      "text": "the optim solution after doing PO is basically<01:29:26.560><c> one</c><01:29:26.760><c> model</c><01:29:27.119><c> that</c><01:29:27.320><c> gives</c><01:29:27.560><c> you</c><01:29:28.119><c> uh</c>"
+    },
+    {
+      "start": 5368.55,
+      "duration": 0.0,
+      "text": "basically one model that gives you uh"
+    },
+    {
+      "start": 5368.56,
+      "duration": 0.0,
+      "text": "basically one model that gives you uh essentially<01:29:28.960><c> a</c><01:29:29.400><c> Delta</c><01:29:30.400><c> um</c><01:29:31.119><c> like</c><01:29:31.320><c> basically</c>"
+    },
+    {
+      "start": 5371.669,
+      "duration": 0.0,
+      "text": "essentially a Delta um like basically"
+    },
+    {
+      "start": 5371.679,
+      "duration": 0.0,
+      "text": "essentially a Delta um like basically says<01:29:31.880><c> that</c><01:29:32.040><c> there's</c><01:29:32.199><c> only</c><01:29:32.400><c> one</c><01:29:32.600><c> sentence</c><01:29:33.280><c> that</c>"
+    },
+    {
+      "start": 5373.709,
+      "duration": 0.0,
+      "text": "says that there's only one sentence that"
+    },
+    {
+      "start": 5373.719,
+      "duration": 0.0,
+      "text": "says that there's only one sentence that is<01:29:34.719><c> that</c><01:29:34.880><c> could</c><01:29:35.040><c> be</c><01:29:35.159><c> generated</c><01:29:35.800><c> for</c><01:29:36.040><c> that</c>"
+    },
+    {
+      "start": 5376.229,
+      "duration": 0.0,
+      "text": "is that could be generated for that"
+    },
+    {
+      "start": 5376.239,
+      "duration": 0.0,
+      "text": "is that could be generated for that question<01:29:36.960><c> so</c><01:29:37.159><c> now</c><01:29:37.360><c> if</c><01:29:37.440><c> you</c><01:29:37.600><c> use</c><01:29:37.760><c> it</c><01:29:37.920><c> on</c>"
+    },
+    {
+      "start": 5378.07,
+      "duration": 0.0,
+      "text": "question so now if you use it on"
+    },
+    {
+      "start": 5378.08,
+      "duration": 0.0,
+      "text": "question so now if you use it on something<01:29:38.360><c> that</c><01:29:38.440><c> is</c><01:29:38.600><c> slightly</c><01:29:39.000><c> semantically</c>"
+    },
+    {
+      "start": 5379.55,
+      "duration": 0.0,
+      "text": "something that is slightly semantically"
+    },
+    {
+      "start": 5379.56,
+      "duration": 0.0,
+      "text": "something that is slightly semantically differently<01:29:40.280><c> different</c><01:29:40.840><c> it</c><01:29:40.960><c> would</c><01:29:41.119><c> actually</c>"
+    },
+    {
+      "start": 5381.31,
+      "duration": 0.0,
+      "text": "differently different it would actually"
+    },
+    {
+      "start": 5381.32,
+      "duration": 0.0,
+      "text": "differently different it would actually give<01:29:41.440><c> a</c><01:29:41.560><c> likelihood</c><01:29:42.000><c> of</c><01:29:42.159><c> zero</c><01:29:42.719><c> for</c><01:29:42.960><c> that</c>"
+    },
+    {
+      "start": 5383.39,
+      "duration": 0.0,
+      "text": "give a likelihood of zero for that"
+    },
+    {
+      "start": 5383.4,
+      "duration": 0.0,
+      "text": "give a likelihood of zero for that answer<01:29:44.400><c> so</c><01:29:44.679><c> in</c><01:29:44.880><c> reality</c><01:29:45.239><c> it's</c><01:29:45.360><c> not</c><01:29:45.560><c> that</c>"
+    },
+    {
+      "start": 5385.75,
+      "duration": 0.0,
+      "text": "answer so in reality it's not that"
+    },
+    {
+      "start": 5385.76,
+      "duration": 0.0,
+      "text": "answer so in reality it's not that extreme<01:29:46.280><c> because</c><01:29:46.440><c> as</c><01:29:46.520><c> you</c><01:29:46.639><c> say</c><01:29:46.760><c> it's</c><01:29:46.880><c> still</c><01:29:47.040><c> a</c>"
+    },
+    {
+      "start": 5387.109,
+      "duration": 0.0,
+      "text": "extreme because as you say it's still a"
+    },
+    {
+      "start": 5387.119,
+      "duration": 0.0,
+      "text": "extreme because as you say it's still a distribution<01:29:47.560><c> but</c><01:29:47.679><c> I</c><01:29:47.800><c> just</c><01:29:48.080><c> shows</c><01:29:48.400><c> you</c><01:29:48.600><c> that</c>"
+    },
+    {
+      "start": 5388.75,
+      "duration": 0.0,
+      "text": "distribution but I just shows you that"
+    },
+    {
+      "start": 5388.76,
+      "duration": 0.0,
+      "text": "distribution but I just shows you that there's<01:29:48.880><c> a</c><01:29:49.080><c> there's</c><01:29:49.239><c> a</c><01:29:49.360><c> fundamental</c><01:29:49.880><c> issue</c>"
+    },
+    {
+      "start": 5390.47,
+      "duration": 0.0,
+      "text": "there's a there's a fundamental issue"
+    },
+    {
+      "start": 5390.48,
+      "duration": 0.0,
+      "text": "there's a there's a fundamental issue with<01:29:50.679><c> perplexity</c><01:29:51.600><c> once</c><01:29:51.920><c> these</c><01:29:52.119><c> models</c><01:29:52.760><c> are</c>"
+    },
+    {
+      "start": 5393.03,
+      "duration": 0.0,
+      "text": "with perplexity once these models are"
+    },
+    {
+      "start": 5393.04,
+      "duration": 0.0,
+      "text": "with perplexity once these models are not<01:29:54.040><c> llms</c><01:29:54.560><c> anymore</c><01:29:54.880><c> they</c><01:29:55.000><c> were</c><01:29:55.119><c> not</c><01:29:55.280><c> trained</c>"
+    },
+    {
+      "start": 5396.07,
+      "duration": 0.0,
+      "text": "not llms anymore they were not trained"
+    },
+    {
+      "start": 5396.08,
+      "duration": 0.0,
+      "text": "not llms anymore they were not trained at<01:29:56.159><c> least</c><01:29:56.320><c> with</c><01:29:56.440><c> P</c><01:29:56.800><c> they</c><01:29:56.880><c> were</c><01:29:57.000><c> not</c><01:29:57.119><c> trained</c><01:29:57.360><c> to</c>"
+    },
+    {
+      "start": 5397.55,
+      "duration": 0.0,
+      "text": "at least with P they were not trained to"
+    },
+    {
+      "start": 5397.56,
+      "duration": 0.0,
+      "text": "at least with P they were not trained to to<01:29:57.679><c> do</c><01:29:57.840><c> maximum</c><01:29:58.199><c> likelihood</c><01:29:58.639><c> anymore</c><01:29:59.119><c> they</c>"
+    },
+    {
+      "start": 5399.189,
+      "duration": 0.0,
+      "text": "to do maximum likelihood anymore they"
+    },
+    {
+      "start": 5399.199,
+      "duration": 0.0,
+      "text": "to do maximum likelihood anymore they were<01:29:59.360><c> trained</c><01:29:59.600><c> to</c><01:29:59.679><c> be</c>"
+    },
+    {
+      "start": 5402.36,
+      "duration": 0.0,
+      "text": "policies<01:30:03.360><c> okay</c><01:30:04.040><c> um</c><01:30:04.400><c> so</c><01:30:04.840><c> probably</c><01:30:05.159><c> the</c><01:30:05.280><c> most</c>"
+    },
+    {
+      "start": 5405.55,
+      "duration": 0.0,
+      "text": "policies okay um so probably the most"
+    },
+    {
+      "start": 5405.56,
+      "duration": 0.0,
+      "text": "policies okay um so probably the most common<01:30:06.000><c> or</c><01:30:06.199><c> like</c><01:30:06.400><c> the</c><01:30:06.560><c> most</c><01:30:07.520><c> um</c><01:30:08.520><c> yeah</c><01:30:08.679><c> the</c><01:30:08.760><c> most</c>"
+    },
+    {
+      "start": 5408.95,
+      "duration": 0.0,
+      "text": "common or like the most um yeah the most"
+    },
+    {
+      "start": 5408.96,
+      "duration": 0.0,
+      "text": "common or like the most um yeah the most common<01:30:09.320><c> Benchmark</c><01:30:09.800><c> or</c><01:30:10.080><c> the</c><01:30:10.159><c> most</c><01:30:10.320><c> trusted</c><01:30:10.719><c> one</c>"
+    },
+    {
+      "start": 5410.99,
+      "duration": 0.0,
+      "text": "common Benchmark or the most trusted one"
+    },
+    {
+      "start": 5411.0,
+      "duration": 0.0,
+      "text": "common Benchmark or the most trusted one is<01:30:11.159><c> what</c><01:30:11.280><c> we</c><01:30:11.400><c> call</c><01:30:11.600><c> Chad</c><01:30:11.960><c> uh</c><01:30:12.080><c> sorry</c><01:30:12.280><c> chatbot</c>"
+    },
+    {
+      "start": 5412.709,
+      "duration": 0.0,
+      "text": "is what we call Chad uh sorry chatbot"
+    },
+    {
+      "start": 5412.719,
+      "duration": 0.0,
+      "text": "is what we call Chad uh sorry chatbot Arena<01:30:13.639><c> uh</c><01:30:13.760><c> which</c><01:30:13.880><c> is</c><01:30:14.119><c> basically</c><01:30:14.800><c> go</c><01:30:15.000><c> on</c>"
+    },
+    {
+      "start": 5415.189,
+      "duration": 0.0,
+      "text": "Arena uh which is basically go on"
+    },
+    {
+      "start": 5415.199,
+      "duration": 0.0,
+      "text": "Arena uh which is basically go on internet<01:30:15.760><c> have</c><01:30:16.000><c> random</c><01:30:16.320><c> users</c><01:30:16.679><c> on</c><01:30:16.840><c> the</c>"
+    },
+    {
+      "start": 5416.99,
+      "duration": 0.0,
+      "text": "internet have random users on the"
+    },
+    {
+      "start": 5417.0,
+      "duration": 0.0,
+      "text": "internet have random users on the internet<01:30:17.600><c> blindly</c><01:30:18.520><c> talk</c><01:30:18.800><c> with</c><01:30:19.000><c> two</c><01:30:19.199><c> chat</c><01:30:19.440><c> Bots</c>"
+    },
+    {
+      "start": 5419.95,
+      "duration": 0.0,
+      "text": "internet blindly talk with two chat Bots"
+    },
+    {
+      "start": 5419.96,
+      "duration": 0.0,
+      "text": "internet blindly talk with two chat Bots just<01:30:20.199><c> ask</c><01:30:20.520><c> many</c><01:30:20.800><c> questions</c><01:30:21.320><c> see</c><01:30:21.600><c> the</c><01:30:21.719><c> two</c>"
+    },
+    {
+      "start": 5421.91,
+      "duration": 0.0,
+      "text": "just ask many questions see the two"
+    },
+    {
+      "start": 5421.92,
+      "duration": 0.0,
+      "text": "just ask many questions see the two answers<01:30:22.480><c> and</c><01:30:22.679><c> rate</c><01:30:22.960><c> which</c><01:30:23.080><c> one</c><01:30:23.199><c> is</c><01:30:23.400><c> better</c><01:30:23.800><c> and</c>"
+    },
+    {
+      "start": 5423.99,
+      "duration": 0.0,
+      "text": "answers and rate which one is better and"
+    },
+    {
+      "start": 5424.0,
+      "duration": 0.0,
+      "text": "answers and rate which one is better and and<01:30:24.080><c> you</c><01:30:24.159><c> do</c><01:30:24.320><c> that</c><01:30:24.480><c> over</c><01:30:25.159><c> hundred</c><01:30:25.440><c> of</c>"
+    },
+    {
+      "start": 5425.55,
+      "duration": 0.0,
+      "text": "and you do that over hundred of"
+    },
+    {
+      "start": 5425.56,
+      "duration": 0.0,
+      "text": "and you do that over hundred of thousands<01:30:25.880><c> of</c><01:30:26.040><c> users</c><01:30:26.639><c> and</c><01:30:26.760><c> then</c><01:30:26.880><c> you</c><01:30:27.080><c> get</c><01:30:27.760><c> uh</c>"
+    },
+    {
+      "start": 5427.91,
+      "duration": 0.0,
+      "text": "thousands of users and then you get uh"
+    },
+    {
+      "start": 5427.92,
+      "duration": 0.0,
+      "text": "thousands of users and then you get uh the<01:30:28.080><c> actual</c><01:30:28.360><c> preferences</c><01:30:28.920><c> and</c><01:30:29.040><c> you</c><01:30:29.199><c> get</c>"
+    },
+    {
+      "start": 5429.47,
+      "duration": 0.0,
+      "text": "the actual preferences and you get"
+    },
+    {
+      "start": 5429.48,
+      "duration": 0.0,
+      "text": "the actual preferences and you get rankings<01:30:29.920><c> of</c><01:30:30.080><c> models</c><01:30:30.960><c> uh</c><01:30:31.040><c> so</c><01:30:31.239><c> you</c><01:30:31.320><c> can</c><01:30:31.520><c> go</c>"
+    },
+    {
+      "start": 5431.75,
+      "duration": 0.0,
+      "text": "rankings of models uh so you can go"
+    },
+    {
+      "start": 5431.76,
+      "duration": 0.0,
+      "text": "rankings of models uh so you can go right<01:30:31.960><c> now</c><01:30:32.440><c> on</c><01:30:32.679><c> chatbot</c><01:30:33.119><c> Arena</c><01:30:33.520><c> and</c><01:30:33.679><c> actually</c>"
+    },
+    {
+      "start": 5434.07,
+      "duration": 0.0,
+      "text": "right now on chatbot Arena and actually"
+    },
+    {
+      "start": 5434.08,
+      "duration": 0.0,
+      "text": "right now on chatbot Arena and actually interact<01:30:34.440><c> with</c><01:30:34.600><c> these</c><01:30:34.719><c> models</c><01:30:35.679><c> um</c><01:30:36.199><c> one</c>"
+    },
+    {
+      "start": 5436.55,
+      "duration": 0.0,
+      "text": "interact with these models um one"
+    },
+    {
+      "start": 5436.56,
+      "duration": 0.0,
+      "text": "interact with these models um one potential<01:30:37.000><c> issue</c><01:30:37.480><c> just</c><01:30:37.639><c> to</c><01:30:37.880><c> highlight</c><01:30:38.560><c> is</c>"
+    },
+    {
+      "start": 5438.709,
+      "duration": 0.0,
+      "text": "potential issue just to highlight is"
+    },
+    {
+      "start": 5438.719,
+      "duration": 0.0,
+      "text": "potential issue just to highlight is that<01:30:38.920><c> while</c><01:30:39.119><c> people</c><01:30:39.360><c> who</c><01:30:39.560><c> want</c><01:30:39.679><c> to</c><01:30:39.840><c> do</c><01:30:40.040><c> these</c>"
+    },
+    {
+      "start": 5440.189,
+      "duration": 0.0,
+      "text": "that while people who want to do these"
+    },
+    {
+      "start": 5440.199,
+      "duration": 0.0,
+      "text": "that while people who want to do these type<01:30:40.360><c> of</c><01:30:40.480><c> things</c><01:30:40.600><c> are</c><01:30:40.760><c> usually</c><01:30:41.000><c> more</c><01:30:41.199><c> like</c>"
+    },
+    {
+      "start": 5441.31,
+      "duration": 0.0,
+      "text": "type of things are usually more like"
+    },
+    {
+      "start": 5441.32,
+      "duration": 0.0,
+      "text": "type of things are usually more like Tech<01:30:41.560><c> driven</c><01:30:42.400><c> um</c><01:30:42.560><c> or</c><01:30:42.760><c> like</c><01:30:43.159><c> techsavvy</c><01:30:43.679><c> uh</c><01:30:44.400><c> so</c><01:30:44.639><c> a</c>"
+    },
+    {
+      "start": 5444.709,
+      "duration": 0.0,
+      "text": "Tech driven um or like techsavvy uh so a"
+    },
+    {
+      "start": 5444.719,
+      "duration": 0.0,
+      "text": "Tech driven um or like techsavvy uh so a lot<01:30:44.840><c> of</c><01:30:44.960><c> the</c><01:30:45.040><c> questions</c><01:30:45.320><c> that</c><01:30:45.440><c> you</c><01:30:45.520><c> will</c><01:30:45.679><c> ask</c>"
+    },
+    {
+      "start": 5445.99,
+      "duration": 0.0,
+      "text": "lot of the questions that you will ask"
+    },
+    {
+      "start": 5446.0,
+      "duration": 0.0,
+      "text": "lot of the questions that you will ask are<01:30:46.239><c> more</c><01:30:46.440><c> like</c><01:30:46.639><c> Tech</c><01:30:46.880><c> stuff</c><01:30:47.400><c> discussing</c>"
+    },
+    {
+      "start": 5447.95,
+      "duration": 0.0,
+      "text": "are more like Tech stuff discussing"
+    },
+    {
+      "start": 5447.96,
+      "duration": 0.0,
+      "text": "are more like Tech stuff discussing software<01:30:48.320><c> errors</c><01:30:48.960><c> inquiries</c><01:30:49.440><c> about</c><01:30:49.679><c> AI</c><01:30:49.960><c> tools</c>"
+    },
+    {
+      "start": 5450.31,
+      "duration": 0.0,
+      "text": "software errors inquiries about AI tools"
+    },
+    {
+      "start": 5450.32,
+      "duration": 0.0,
+      "text": "software errors inquiries about AI tools and<01:30:50.480><c> all</c><01:30:50.719><c> these</c><01:30:50.880><c> things</c><01:30:51.960><c> um</c><01:30:52.960><c> so</c><01:30:53.320><c> another</c><01:30:53.560><c> issue</c>"
+    },
+    {
+      "start": 5453.79,
+      "duration": 0.0,
+      "text": "and all these things um so another issue"
+    },
+    {
+      "start": 5453.8,
+      "duration": 0.0,
+      "text": "and all these things um so another issue is<01:30:54.119><c> cost</c><01:30:54.320><c> and</c><01:30:54.480><c> speed</c><01:30:54.840><c> if</c><01:30:54.960><c> you</c><01:30:55.080><c> really</c><01:30:55.239><c> want</c><01:30:55.360><c> to</c>"
+    },
+    {
+      "start": 5455.51,
+      "duration": 0.0,
+      "text": "is cost and speed if you really want to"
+    },
+    {
+      "start": 5455.52,
+      "duration": 0.0,
+      "text": "is cost and speed if you really want to use<01:30:55.719><c> something</c><01:30:56.000><c> like</c><01:30:56.199><c> this</c><01:30:56.480><c> for</c><01:30:56.760><c> development</c>"
+    },
+    {
+      "start": 5457.35,
+      "duration": 0.0,
+      "text": "use something like this for development"
+    },
+    {
+      "start": 5457.36,
+      "duration": 0.0,
+      "text": "use something like this for development process<01:30:58.360><c> um</c><01:30:58.719><c> it</c><01:30:58.840><c> will</c><01:30:58.960><c> be</c><01:30:59.119><c> too</c><01:30:59.360><c> costly</c><01:30:59.840><c> because</c>"
+    },
+    {
+      "start": 5459.95,
+      "duration": 0.0,
+      "text": "process um it will be too costly because"
+    },
+    {
+      "start": 5459.96,
+      "duration": 0.0,
+      "text": "process um it will be too costly because you<01:31:00.080><c> would</c><01:31:00.239><c> need</c><01:31:00.400><c> to</c><01:31:00.960><c> basically</c><01:31:01.320><c> pay</c><01:31:01.480><c> a</c><01:31:01.560><c> lot</c><01:31:01.639><c> of</c>"
+    },
+    {
+      "start": 5461.75,
+      "duration": 0.0,
+      "text": "you would need to basically pay a lot of"
+    },
+    {
+      "start": 5461.76,
+      "duration": 0.0,
+      "text": "you would need to basically pay a lot of humans<01:31:02.040><c> to</c><01:31:02.199><c> do</c><01:31:02.920><c> that</c><01:31:03.920><c> so</c><01:31:04.159><c> one</c><01:31:04.400><c> simple</c><01:31:04.760><c> idea</c><01:31:05.600><c> is</c>"
+    },
+    {
+      "start": 5466.07,
+      "duration": 0.0,
+      "text": "humans to do that so one simple idea is"
+    },
+    {
+      "start": 5466.08,
+      "duration": 0.0,
+      "text": "humans to do that so one simple idea is again<01:31:06.480><c> as</c><01:31:06.719><c> we</c><01:31:06.960><c> said</c><01:31:07.199><c> many</c><01:31:07.480><c> times</c><01:31:07.920><c> just</c><01:31:08.080><c> use</c><01:31:08.320><c> LM</c>"
+    },
+    {
+      "start": 5468.75,
+      "duration": 0.0,
+      "text": "again as we said many times just use LM"
+    },
+    {
+      "start": 5468.76,
+      "duration": 0.0,
+      "text": "again as we said many times just use LM instead<01:31:09.000><c> of</c><01:31:09.199><c> humans</c><01:31:10.199><c> uh</c><01:31:10.440><c> you</c><01:31:10.639><c> probably</c><01:31:10.880><c> know</c>"
+    },
+    {
+      "start": 5471.07,
+      "duration": 0.0,
+      "text": "instead of humans uh you probably know"
+    },
+    {
+      "start": 5471.08,
+      "duration": 0.0,
+      "text": "instead of humans uh you probably know the<01:31:11.199><c> drill</c><01:31:11.719><c> at</c><01:31:11.880><c> this</c><01:31:12.040><c> point</c><01:31:12.920><c> uh</c><01:31:13.080><c> steps</c><01:31:13.520><c> for</c>"
+    },
+    {
+      "start": 5473.83,
+      "duration": 0.0,
+      "text": "the drill at this point uh steps for"
+    },
+    {
+      "start": 5473.84,
+      "duration": 0.0,
+      "text": "the drill at this point uh steps for every<01:31:14.119><c> instruction</c><01:31:14.639><c> generate</c><01:31:15.199><c> outputs</c><01:31:15.760><c> by</c>"
+    },
+    {
+      "start": 5475.95,
+      "duration": 0.0,
+      "text": "every instruction generate outputs by"
+    },
+    {
+      "start": 5475.96,
+      "duration": 0.0,
+      "text": "every instruction generate outputs by some<01:31:16.199><c> baseline</c><01:31:17.000><c> and</c><01:31:17.159><c> the</c><01:31:17.320><c> model</c><01:31:17.600><c> that</c><01:31:17.679><c> you</c>"
+    },
+    {
+      "start": 5477.75,
+      "duration": 0.0,
+      "text": "some baseline and the model that you"
+    },
+    {
+      "start": 5477.76,
+      "duration": 0.0,
+      "text": "some baseline and the model that you want<01:31:17.880><c> to</c><01:31:18.280><c> evaluate</c><01:31:19.280><c> um</c><01:31:19.480><c> so</c><01:31:19.719><c> here</c><01:31:19.880><c> you</c><01:31:20.040><c> imagine</c>"
+    },
+    {
+      "start": 5480.39,
+      "duration": 0.0,
+      "text": "want to evaluate um so here you imagine"
+    },
+    {
+      "start": 5480.4,
+      "duration": 0.0,
+      "text": "want to evaluate um so here you imagine that<01:31:20.679><c> I</c><01:31:20.960><c> I'm</c><01:31:21.199><c> comparing</c><01:31:21.880><c> an</c><01:31:22.080><c> answer</c><01:31:22.440><c> from</c><01:31:22.600><c> Chad</c>"
+    },
+    {
+      "start": 5482.87,
+      "duration": 0.0,
+      "text": "that I I'm comparing an answer from Chad"
+    },
+    {
+      "start": 5482.88,
+      "duration": 0.0,
+      "text": "that I I'm comparing an answer from Chad GPT<01:31:23.440><c> and</c><01:31:23.560><c> from</c>"
+    },
+    {
+      "start": 5484.629,
+      "duration": 0.0,
+      "text": "GPT and from"
+    },
+    {
+      "start": 5484.639,
+      "duration": 0.0,
+      "text": "GPT and from I'm<01:31:24.840><c> just</c><01:31:25.080><c> asking</c><01:31:25.400><c> a</c><01:31:25.560><c> model</c><01:31:26.520><c> uh</c><01:31:26.719><c> another</c><01:31:27.119><c> model</c>"
+    },
+    {
+      "start": 5487.99,
+      "duration": 0.0,
+      "text": "I'm just asking a model uh another model"
+    },
+    {
+      "start": 5488.0,
+      "duration": 0.0,
+      "text": "I'm just asking a model uh another model uh<01:31:28.159><c> which</c><01:31:28.320><c> one</c><01:31:28.480><c> is</c><01:31:28.679><c> better</c><01:31:29.520><c> and</c><01:31:29.800><c> I</c><01:31:30.000><c> just</c>"
+    },
+    {
+      "start": 5490.59,
+      "duration": 0.0,
+      "text": "uh which one is better and I just"
+    },
+    {
+      "start": 5490.6,
+      "duration": 0.0,
+      "text": "uh which one is better and I just basically<01:31:31.040><c> average</c><01:31:31.440><c> that</c><01:31:31.600><c> out</c><01:31:32.320><c> uh</c><01:31:32.480><c> yeah</c><01:31:32.600><c> I</c>"
+    },
+    {
+      "start": 5492.709,
+      "duration": 0.0,
+      "text": "basically average that out uh yeah I"
+    },
+    {
+      "start": 5492.719,
+      "duration": 0.0,
+      "text": "basically average that out uh yeah I asked<01:31:33.159><c> gp4</c><01:31:33.679><c> which</c><01:31:33.800><c> one</c><01:31:33.920><c> is</c><01:31:34.040><c> better</c><01:31:34.520><c> I</c><01:31:34.639><c> average</c>"
+    },
+    {
+      "start": 5494.99,
+      "duration": 0.0,
+      "text": "asked gp4 which one is better I average"
+    },
+    {
+      "start": 5495.0,
+      "duration": 0.0,
+      "text": "asked gp4 which one is better I average that<01:31:35.159><c> out</c><01:31:35.320><c> over</c><01:31:35.639><c> my</c><01:31:35.760><c> entire</c><01:31:36.199><c> distribution</c>"
+    },
+    {
+      "start": 5497.189,
+      "duration": 0.0,
+      "text": "that out over my entire distribution"
+    },
+    {
+      "start": 5497.199,
+      "duration": 0.0,
+      "text": "that out over my entire distribution over<01:31:37.400><c> my</c><01:31:37.600><c> entire</c><01:31:37.920><c> Benchmark</c><01:31:38.360><c> or</c><01:31:38.560><c> data</c><01:31:38.800><c> set</c><01:31:39.280><c> and</c>"
+    },
+    {
+      "start": 5499.55,
+      "duration": 0.0,
+      "text": "over my entire Benchmark or data set and"
+    },
+    {
+      "start": 5499.56,
+      "duration": 0.0,
+      "text": "over my entire Benchmark or data set and that<01:31:39.719><c> gives</c><01:31:39.960><c> me</c><01:31:40.159><c> a</c><01:31:40.560><c> RN</c><01:31:40.840><c> rate</c><01:31:41.080><c> so</c><01:31:41.360><c> RN</c>"
+    },
+    {
+      "start": 5501.629,
+      "duration": 0.0,
+      "text": "that gives me a RN rate so RN"
+    },
+    {
+      "start": 5501.639,
+      "duration": 0.0,
+      "text": "that gives me a RN rate so RN probability<01:31:42.440><c> for</c><01:31:42.679><c> one</c><01:31:42.880><c> model</c><01:31:43.520><c> compared</c><01:31:43.800><c> to</c>"
+    },
+    {
+      "start": 5503.91,
+      "duration": 0.0,
+      "text": "probability for one model compared to"
+    },
+    {
+      "start": 5503.92,
+      "duration": 0.0,
+      "text": "probability for one model compared to another<01:31:44.159><c> one</c><01:31:44.600><c> and</c><01:31:44.840><c> now</c><01:31:44.960><c> you</c><01:31:45.040><c> can</c><01:31:45.199><c> rank</c><01:31:45.719><c> models</c>"
+    },
+    {
+      "start": 5506.709,
+      "duration": 0.0,
+      "text": "another one and now you can rank models"
+    },
+    {
+      "start": 5506.719,
+      "duration": 0.0,
+      "text": "another one and now you can rank models uh<01:31:46.840><c> and</c><01:31:46.960><c> this</c><01:31:47.080><c> is</c><01:31:47.199><c> the</c><01:31:47.320><c> Alpa</c><01:31:47.760><c> eval</c><01:31:48.520><c> uh</c>"
+    },
+    {
+      "start": 5509.51,
+      "duration": 0.0,
+      "text": "uh and this is the Alpa eval uh"
+    },
+    {
+      "start": 5509.52,
+      "duration": 0.0,
+      "text": "uh and this is the Alpa eval uh leaderboard<01:31:50.520><c> so</c><01:31:50.880><c> the</c><01:31:51.000><c> benefits</c><01:31:51.400><c> of</c><01:31:51.639><c> this</c><01:31:52.000><c> is</c>"
+    },
+    {
+      "start": 5512.189,
+      "duration": 0.0,
+      "text": "leaderboard so the benefits of this is"
+    },
+    {
+      "start": 5512.199,
+      "duration": 0.0,
+      "text": "leaderboard so the benefits of this is that<01:31:52.440><c> actually</c><01:31:52.800><c> we</c><01:31:52.960><c> show</c><01:31:53.400><c> we</c><01:31:53.560><c> get</c><01:31:53.920><c> 98%</c>"
+    },
+    {
+      "start": 5514.55,
+      "duration": 0.0,
+      "text": "that actually we show we get 98%"
+    },
+    {
+      "start": 5514.56,
+      "duration": 0.0,
+      "text": "that actually we show we get 98% correlation<01:31:55.000><c> with</c><01:31:55.119><c> Chad</c><01:31:55.360><c> B</c><01:31:55.520><c> Arena</c><01:31:55.920><c> so</c><01:31:56.159><c> very</c>"
+    },
+    {
+      "start": 5516.35,
+      "duration": 0.0,
+      "text": "correlation with Chad B Arena so very"
+    },
+    {
+      "start": 5516.36,
+      "duration": 0.0,
+      "text": "correlation with Chad B Arena so very high<01:31:56.560><c> correlation</c><01:31:57.040><c> with</c><01:31:57.280><c> humans</c><01:31:58.360><c> um</c><01:31:59.360><c> so</c><01:31:59.639><c> this</c>"
+    },
+    {
+      "start": 5519.79,
+      "duration": 0.0,
+      "text": "high correlation with humans um so this"
+    },
+    {
+      "start": 5519.8,
+      "duration": 0.0,
+      "text": "high correlation with humans um so this is<01:32:00.360><c> yeah</c><01:32:00.679><c> comparison</c><01:32:01.159><c> with</c><01:32:01.239><c> correlation</c><01:32:01.639><c> with</c>"
+    },
+    {
+      "start": 5521.709,
+      "duration": 0.0,
+      "text": "is yeah comparison with correlation with"
+    },
+    {
+      "start": 5521.719,
+      "duration": 0.0,
+      "text": "is yeah comparison with correlation with other<01:32:01.960><c> benchmarks</c><01:32:02.920><c> and</c><01:32:03.040><c> it</c><01:32:03.199><c> takes</c><01:32:03.440><c> less</c><01:32:03.600><c> than</c>"
+    },
+    {
+      "start": 5523.75,
+      "duration": 0.0,
+      "text": "other benchmarks and it takes less than"
+    },
+    {
+      "start": 5523.76,
+      "duration": 0.0,
+      "text": "other benchmarks and it takes less than three<01:32:03.920><c> minutes</c><01:32:04.199><c> and</c><01:32:04.400><c> less</c><01:32:04.560><c> than</c><01:32:04.639><c> $10</c><01:32:05.199><c> to</c><01:32:05.320><c> run</c>"
+    },
+    {
+      "start": 5525.51,
+      "duration": 0.0,
+      "text": "three minutes and less than $10 to run"
+    },
+    {
+      "start": 5525.52,
+      "duration": 0.0,
+      "text": "three minutes and less than $10 to run so<01:32:05.639><c> it's</c><01:32:05.760><c> pretty</c><01:32:06.040><c> cheap</c><01:32:07.040><c> um</c><01:32:07.280><c> there</c><01:32:07.400><c> are</c>"
+    },
+    {
+      "start": 5527.55,
+      "duration": 0.0,
+      "text": "so it's pretty cheap um there are"
+    },
+    {
+      "start": 5527.56,
+      "duration": 0.0,
+      "text": "so it's pretty cheap um there are downsides<01:32:08.199><c> though</c><01:32:08.760><c> uh</c><01:32:08.880><c> one</c><01:32:09.000><c> of</c><01:32:09.159><c> them</c><01:32:09.400><c> is</c><01:32:09.600><c> purus</c>"
+    },
+    {
+      "start": 5530.31,
+      "duration": 0.0,
+      "text": "downsides though uh one of them is purus"
+    },
+    {
+      "start": 5530.32,
+      "duration": 0.0,
+      "text": "downsides though uh one of them is purus correlation<01:32:11.320><c> um</c><01:32:11.719><c> so</c><01:32:12.119><c> as</c><01:32:12.239><c> we</c><01:32:12.400><c> already</c><01:32:12.639><c> saw</c>"
+    },
+    {
+      "start": 5532.99,
+      "duration": 0.0,
+      "text": "correlation um so as we already saw"
+    },
+    {
+      "start": 5533.0,
+      "duration": 0.0,
+      "text": "correlation um so as we already saw before<01:32:13.960><c> LMS</c><01:32:14.520><c> prefer</c><01:32:15.040><c> this</c><01:32:15.119><c> is</c><01:32:15.239><c> one</c><01:32:15.400><c> SP</c>"
+    },
+    {
+      "start": 5535.75,
+      "duration": 0.0,
+      "text": "before LMS prefer this is one SP"
+    },
+    {
+      "start": 5535.76,
+      "duration": 0.0,
+      "text": "before LMS prefer this is one SP correlation<01:32:16.119><c> not</c><01:32:16.280><c> many</c><01:32:16.520><c> I'll</c><01:32:16.639><c> just</c><01:32:16.960><c> talk</c>"
+    },
+    {
+      "start": 5537.109,
+      "duration": 0.0,
+      "text": "correlation not many I'll just talk"
+    },
+    {
+      "start": 5537.119,
+      "duration": 0.0,
+      "text": "correlation not many I'll just talk about<01:32:17.320><c> one</c><01:32:17.560><c> LMS</c><01:32:18.000><c> prefer</c><01:32:18.239><c> longer</c><01:32:18.520><c> outputs</c>"
+    },
+    {
+      "start": 5538.95,
+      "duration": 0.0,
+      "text": "about one LMS prefer longer outputs"
+    },
+    {
+      "start": 5538.96,
+      "duration": 0.0,
+      "text": "about one LMS prefer longer outputs actually<01:32:19.199><c> humans</c><01:32:19.520><c> also</c><01:32:19.719><c> prefer</c><01:32:20.000><c> longer</c>"
+    },
+    {
+      "start": 5540.27,
+      "duration": 0.0,
+      "text": "actually humans also prefer longer"
+    },
+    {
+      "start": 5540.28,
+      "duration": 0.0,
+      "text": "actually humans also prefer longer outputs<01:32:21.000><c> but</c><01:32:21.119><c> the</c><01:32:21.320><c> problem</c><01:32:21.840><c> or</c><01:32:22.040><c> the</c><01:32:22.199><c> issue</c>"
+    },
+    {
+      "start": 5542.43,
+      "duration": 0.0,
+      "text": "outputs but the problem or the issue"
+    },
+    {
+      "start": 5542.44,
+      "duration": 0.0,
+      "text": "outputs but the problem or the issue once<01:32:22.600><c> you</c><01:32:22.679><c> use</c><01:32:22.840><c> llms</c><01:32:23.320><c> is</c><01:32:23.400><c> that</c><01:32:23.520><c> once</c><01:32:23.679><c> there</c>"
+    },
+    {
+      "start": 5544.07,
+      "duration": 0.0,
+      "text": "once you use llms is that once there"
+    },
+    {
+      "start": 5544.08,
+      "duration": 0.0,
+      "text": "once you use llms is that once there bias<01:32:24.560><c> you</c><01:32:24.679><c> will</c><01:32:24.880><c> continue</c><01:32:25.239><c> optimizing</c><01:32:25.800><c> that</c>"
+    },
+    {
+      "start": 5546.35,
+      "duration": 0.0,
+      "text": "bias you will continue optimizing that"
+    },
+    {
+      "start": 5546.36,
+      "duration": 0.0,
+      "text": "bias you will continue optimizing that humans<01:32:26.719><c> at</c><01:32:26.840><c> some</c><01:32:27.000><c> point</c><01:32:27.360><c> I</c><01:32:27.440><c> can</c><01:32:27.600><c> guarantee</c><01:32:27.960><c> you</c>"
+    },
+    {
+      "start": 5548.07,
+      "duration": 0.0,
+      "text": "humans at some point I can guarantee you"
+    },
+    {
+      "start": 5548.08,
+      "duration": 0.0,
+      "text": "humans at some point I can guarantee you if<01:32:28.159><c> I</c><01:32:28.280><c> ask</c><01:32:28.440><c> a</c><01:32:28.560><c> simple</c><01:32:28.840><c> question</c><01:32:29.080><c> and</c><01:32:29.199><c> you</c><01:32:29.320><c> give</c>"
+    },
+    {
+      "start": 5549.47,
+      "duration": 0.0,
+      "text": "if I ask a simple question and you give"
+    },
+    {
+      "start": 5549.48,
+      "duration": 0.0,
+      "text": "if I ask a simple question and you give me<01:32:29.719><c> five</c><01:32:29.960><c> pages</c><01:32:30.239><c> of</c><01:32:30.440><c> answers</c><01:32:30.960><c> I'll</c><01:32:31.119><c> be</c><01:32:31.239><c> like</c><01:32:31.360><c> no</c>"
+    },
+    {
+      "start": 5551.47,
+      "duration": 0.0,
+      "text": "me five pages of answers I'll be like no"
+    },
+    {
+      "start": 5551.48,
+      "duration": 0.0,
+      "text": "me five pages of answers I'll be like no I<01:32:31.520><c> don't</c><01:32:31.679><c> like</c><01:32:31.800><c> that</c><01:32:32.000><c> answer</c><01:32:32.520><c> but</c><01:32:32.679><c> LMS</c><01:32:33.119><c> if</c><01:32:33.239><c> they</c>"
+    },
+    {
+      "start": 5553.39,
+      "duration": 0.0,
+      "text": "I don't like that answer but LMS if they"
+    },
+    {
+      "start": 5553.4,
+      "duration": 0.0,
+      "text": "I don't like that answer but LMS if they have<01:32:33.560><c> this</c><01:32:33.719><c> bius</c><01:32:34.040><c> and</c><01:32:34.119><c> they</c><01:32:34.199><c> were</c><01:32:34.320><c> trained</c><01:32:34.600><c> for</c>"
+    },
+    {
+      "start": 5554.75,
+      "duration": 0.0,
+      "text": "have this bius and they were trained for"
+    },
+    {
+      "start": 5554.76,
+      "duration": 0.0,
+      "text": "have this bius and they were trained for that<01:32:35.159><c> they</c><01:32:35.239><c> will</c><01:32:35.400><c> continue</c><01:32:35.760><c> preferring</c>"
+    },
+    {
+      "start": 5556.149,
+      "duration": 0.0,
+      "text": "that they will continue preferring"
+    },
+    {
+      "start": 5556.159,
+      "duration": 0.0,
+      "text": "that they will continue preferring longer<01:32:36.840><c> outputs</c><01:32:37.840><c> so</c><01:32:38.800><c> uh</c><01:32:38.960><c> here</c><01:32:39.159><c> we</c><01:32:39.320><c> see</c><01:32:40.159><c> um</c><01:32:41.159><c> the</c>"
+    },
+    {
+      "start": 5561.47,
+      "duration": 0.0,
+      "text": "longer outputs so uh here we see um the"
+    },
+    {
+      "start": 5561.48,
+      "duration": 0.0,
+      "text": "longer outputs so uh here we see um the the<01:32:41.600><c> preference</c><01:32:42.320><c> just</c><01:32:42.440><c> showing</c><01:32:42.800><c> that</c><01:32:42.960><c> like</c>"
+    },
+    {
+      "start": 5563.109,
+      "duration": 0.0,
+      "text": "the preference just showing that like"
+    },
+    {
+      "start": 5563.119,
+      "duration": 0.0,
+      "text": "the preference just showing that like humans<01:32:43.480><c> and</c><01:32:43.679><c> models</c><01:32:44.080><c> prefer</c><01:32:44.360><c> longer</c><01:32:44.960><c> outputs</c>"
+    },
+    {
+      "start": 5565.95,
+      "duration": 0.0,
+      "text": "humans and models prefer longer outputs"
+    },
+    {
+      "start": 5565.96,
+      "duration": 0.0,
+      "text": "humans and models prefer longer outputs um<01:32:46.480><c> and</c><01:32:46.719><c> here</c><01:32:46.840><c> is</c><01:32:47.080><c> another</c><01:32:47.639><c> view</c><01:32:48.199><c> of</c><01:32:48.360><c> the</c>"
+    },
+    {
+      "start": 5568.47,
+      "duration": 0.0,
+      "text": "um and here is another view of the"
+    },
+    {
+      "start": 5568.48,
+      "duration": 0.0,
+      "text": "um and here is another view of the initial<01:32:48.800><c> apaka</c><01:32:49.199><c> eval</c><01:32:49.520><c> data</c><01:32:50.159><c> uh</c><01:32:50.360><c> Benchmark</c>"
+    },
+    {
+      "start": 5571.27,
+      "duration": 0.0,
+      "text": "initial apaka eval data uh Benchmark"
+    },
+    {
+      "start": 5571.28,
+      "duration": 0.0,
+      "text": "initial apaka eval data uh Benchmark where<01:32:51.520><c> when</c><01:32:51.679><c> we</c><01:32:51.960><c> asked</c><01:32:52.960><c> um</c><01:32:53.520><c> when</c><01:32:53.639><c> we</c><01:32:53.840><c> we</c><01:32:53.960><c> rank</c>"
+    },
+    {
+      "start": 5574.27,
+      "duration": 0.0,
+      "text": "where when we asked um when we we rank"
+    },
+    {
+      "start": 5574.28,
+      "duration": 0.0,
+      "text": "where when we asked um when we we rank gp4<01:32:55.159><c> when</c><01:32:55.320><c> we</c><01:32:55.440><c> look</c><01:32:55.560><c> at</c><01:32:55.679><c> the</c><01:32:55.800><c> Run</c><01:32:56.000><c> rate</c><01:32:56.199><c> of</c><01:32:56.320><c> gp4</c>"
+    },
+    {
+      "start": 5576.95,
+      "duration": 0.0,
+      "text": "gp4 when we look at the Run rate of gp4"
+    },
+    {
+      "start": 5576.96,
+      "duration": 0.0,
+      "text": "gp4 when we look at the Run rate of gp4 versus<01:32:57.679><c> actually</c><01:32:58.320><c> uh</c><01:32:58.440><c> gp4</c><01:32:59.119><c> itself</c><01:32:59.719><c> if</c><01:32:59.880><c> we</c><01:33:00.080><c> com</c>"
+    },
+    {
+      "start": 5580.27,
+      "duration": 0.0,
+      "text": "versus actually uh gp4 itself if we com"
+    },
+    {
+      "start": 5580.28,
+      "duration": 0.0,
+      "text": "versus actually uh gp4 itself if we com if<01:33:00.400><c> we</c><01:33:00.480><c> use</c><01:33:00.639><c> the</c><01:33:00.760><c> standard</c><01:33:01.080><c> GPT</c><01:33:01.360><c> 4</c><01:33:01.520><c> it</c><01:33:01.600><c> gets</c><01:33:01.840><c> 50%</c>"
+    },
+    {
+      "start": 5582.59,
+      "duration": 0.0,
+      "text": "if we use the standard GPT 4 it gets 50%"
+    },
+    {
+      "start": 5582.6,
+      "duration": 0.0,
+      "text": "if we use the standard GPT 4 it gets 50% kind<01:33:02.719><c> of</c><01:33:02.840><c> by</c><01:33:03.000><c> definition</c><01:33:03.440><c> because</c><01:33:03.600><c> we're</c>"
+    },
+    {
+      "start": 5583.75,
+      "duration": 0.0,
+      "text": "kind of by definition because we're"
+    },
+    {
+      "start": 5583.76,
+      "duration": 0.0,
+      "text": "kind of by definition because we're comparing<01:33:04.280><c> GPT</c><01:33:04.719><c> 4</c><01:33:05.000><c> versus</c><01:33:05.280><c> gp4</c><01:33:06.080><c> but</c><01:33:06.239><c> if</c><01:33:06.400><c> we</c><01:33:06.679><c> ask</c>"
+    },
+    {
+      "start": 5587.27,
+      "duration": 0.0,
+      "text": "comparing GPT 4 versus gp4 but if we ask"
+    },
+    {
+      "start": 5587.28,
+      "duration": 0.0,
+      "text": "comparing GPT 4 versus gp4 but if we ask a<01:33:07.440><c> gbd4</c><01:33:08.000><c> to</c><01:33:08.080><c> be</c><01:33:08.199><c> slightly</c><01:33:08.520><c> more</c><01:33:08.679><c> verose</c><01:33:09.159><c> so</c><01:33:09.320><c> we</c>"
+    },
+    {
+      "start": 5589.47,
+      "duration": 0.0,
+      "text": "a gbd4 to be slightly more verose so we"
+    },
+    {
+      "start": 5589.48,
+      "duration": 0.0,
+      "text": "a gbd4 to be slightly more verose so we just<01:33:09.600><c> say</c><01:33:09.920><c> in</c><01:33:10.000><c> the</c><01:33:10.159><c> prompt</c><01:33:10.600><c> be</c><01:33:10.880><c> Vos</c><01:33:11.280><c> in</c><01:33:11.360><c> your</c>"
+    },
+    {
+      "start": 5591.51,
+      "duration": 0.0,
+      "text": "just say in the prompt be Vos in your"
+    },
+    {
+      "start": 5591.52,
+      "duration": 0.0,
+      "text": "just say in the prompt be Vos in your answers<01:33:12.199><c> then</c><01:33:12.320><c> it</c><01:33:12.400><c> gets</c><01:33:12.560><c> a</c><01:33:12.679><c> r</c><01:33:12.880><c> rate</c><01:33:13.080><c> of</c>"
+    },
+    {
+      "start": 5593.87,
+      "duration": 0.0,
+      "text": "answers then it gets a r rate of"
+    },
+    {
+      "start": 5593.88,
+      "duration": 0.0,
+      "text": "answers then it gets a r rate of 64.4%<01:33:14.880><c> so</c><01:33:15.159><c> really</c><01:33:15.600><c> there's</c><01:33:15.760><c> a</c><01:33:15.960><c> huge</c><01:33:16.239><c> variance</c>"
+    },
+    {
+      "start": 5596.59,
+      "duration": 0.0,
+      "text": "64.4% so really there's a huge variance"
+    },
+    {
+      "start": 5596.6,
+      "duration": 0.0,
+      "text": "64.4% so really there's a huge variance and<01:33:16.719><c> if</c><01:33:16.800><c> we</c><01:33:16.920><c> ask</c><01:33:17.040><c> it</c><01:33:17.159><c> to</c><01:33:17.239><c> be</c><01:33:17.360><c> concise</c><01:33:17.719><c> it</c><01:33:17.840><c> gets</c>"
+    },
+    {
+      "start": 5598.03,
+      "duration": 0.0,
+      "text": "and if we ask it to be concise it gets"
+    },
+    {
+      "start": 5598.04,
+      "duration": 0.0,
+      "text": "and if we ask it to be concise it gets 20%<01:33:18.800><c> so</c><01:33:18.960><c> there's</c><01:33:19.119><c> a</c><01:33:19.280><c> huge</c><01:33:19.560><c> variance</c><01:33:20.199><c> depending</c>"
+    },
+    {
+      "start": 5600.669,
+      "duration": 0.0,
+      "text": "20% so there's a huge variance depending"
+    },
+    {
+      "start": 5600.679,
+      "duration": 0.0,
+      "text": "20% so there's a huge variance depending on<01:33:21.480><c> um</c><01:33:22.280><c> whether</c><01:33:22.480><c> you</c><01:33:22.639><c> ask</c><01:33:22.800><c> it</c><01:33:22.880><c> to</c><01:33:22.960><c> be</c><01:33:23.080><c> concise</c>"
+    },
+    {
+      "start": 5603.39,
+      "duration": 0.0,
+      "text": "on um whether you ask it to be concise"
+    },
+    {
+      "start": 5603.4,
+      "duration": 0.0,
+      "text": "on um whether you ask it to be concise of"
+    },
+    {
+      "start": 5604.27,
+      "duration": 0.0,
+      "text": "of"
+    },
+    {
+      "start": 5604.28,
+      "duration": 0.0,
+      "text": "of that's<01:33:24.480><c> very</c><01:33:24.800><c> annoying</c><01:33:25.800><c> um</c><01:33:26.159><c> so</c><01:33:26.840><c> one</c><01:33:27.119><c> possible</c>"
+    },
+    {
+      "start": 5607.51,
+      "duration": 0.0,
+      "text": "that's very annoying um so one possible"
+    },
+    {
+      "start": 5607.52,
+      "duration": 0.0,
+      "text": "that's very annoying um so one possible solution<01:33:28.119><c> which</c><01:33:28.280><c> is</c><01:33:28.440><c> what</c><01:33:28.600><c> we</c><01:33:28.800><c> did</c><01:33:29.320><c> is</c><01:33:29.679><c> uh</c><01:33:29.800><c> just</c>"
+    },
+    {
+      "start": 5609.91,
+      "duration": 0.0,
+      "text": "solution which is what we did is uh just"
+    },
+    {
+      "start": 5609.92,
+      "duration": 0.0,
+      "text": "solution which is what we did is uh just use<01:33:30.159><c> some</c><01:33:30.320><c> regression</c><01:33:30.840><c> analysis</c><01:33:31.480><c> I'm</c><01:33:31.560><c> not</c>"
+    },
+    {
+      "start": 5611.75,
+      "duration": 0.0,
+      "text": "use some regression analysis I'm not"
+    },
+    {
+      "start": 5611.76,
+      "duration": 0.0,
+      "text": "use some regression analysis I'm not going<01:33:31.840><c> to</c><01:33:32.000><c> go</c><01:33:32.159><c> into</c><01:33:32.400><c> details</c><01:33:32.760><c> but</c><01:33:32.920><c> basically</c>"
+    },
+    {
+      "start": 5613.229,
+      "duration": 0.0,
+      "text": "going to go into details but basically"
+    },
+    {
+      "start": 5613.239,
+      "duration": 0.0,
+      "text": "going to go into details but basically use<01:33:33.520><c> Cal</c><01:33:33.840><c> inference</c><01:33:34.199><c> tools</c><01:33:34.719><c> to</c><01:33:34.880><c> control</c><01:33:35.199><c> for</c>"
+    },
+    {
+      "start": 5615.39,
+      "duration": 0.0,
+      "text": "use Cal inference tools to control for"
+    },
+    {
+      "start": 5615.4,
+      "duration": 0.0,
+      "text": "use Cal inference tools to control for length<01:33:36.080><c> and</c><01:33:36.320><c> right</c><01:33:36.560><c> now</c><01:33:37.199><c> uh</c><01:33:37.440><c> actually</c><01:33:37.679><c> length</c>"
+    },
+    {
+      "start": 5617.95,
+      "duration": 0.0,
+      "text": "length and right now uh actually length"
+    },
+    {
+      "start": 5617.96,
+      "duration": 0.0,
+      "text": "length and right now uh actually length matters<01:33:38.320><c> much</c><01:33:38.520><c> less</c><01:33:38.800><c> so</c><01:33:39.239><c> if</c><01:33:39.360><c> you</c><01:33:39.480><c> ask</c><01:33:39.639><c> it</c><01:33:39.760><c> to</c><01:33:39.840><c> be</c>"
+    },
+    {
+      "start": 5619.95,
+      "duration": 0.0,
+      "text": "matters much less so if you ask it to be"
+    },
+    {
+      "start": 5619.96,
+      "duration": 0.0,
+      "text": "matters much less so if you ask it to be veros<01:33:40.360><c> we</c><01:33:40.480><c> still</c><01:33:40.679><c> get</c><01:33:40.840><c> some</c><01:33:41.080><c> gains</c><01:33:41.679><c> but</c><01:33:41.920><c> much</c>"
+    },
+    {
+      "start": 5623.59,
+      "duration": 0.0,
+      "text": "veros we still get some gains but much"
+    },
+    {
+      "start": 5623.6,
+      "duration": 0.0,
+      "text": "veros we still get some gains but much less<01:33:44.600><c> great</c><01:33:45.000><c> so</c><01:33:45.239><c> that's</c><01:33:45.480><c> all</c><01:33:45.679><c> about</c><01:33:45.920><c> post</c>"
+    },
+    {
+      "start": 5626.189,
+      "duration": 0.0,
+      "text": "less great so that's all about post"
+    },
+    {
+      "start": 5626.199,
+      "duration": 0.0,
+      "text": "less great so that's all about post training<01:33:46.760><c> and</c><01:33:46.960><c> now</c><01:33:47.159><c> for</c><01:33:47.360><c> the</c><01:33:47.520><c> next</c><01:33:47.920><c> eight</c>"
+    },
+    {
+      "start": 5628.149,
+      "duration": 0.0,
+      "text": "training and now for the next eight"
+    },
+    {
+      "start": 5628.159,
+      "duration": 0.0,
+      "text": "training and now for the next eight minutes<01:33:48.840><c> I</c><01:33:49.040><c> might</c><01:33:49.239><c> talk</c><01:33:49.440><c> about</c><01:33:49.719><c> systems</c><01:33:50.199><c> or</c>"
+    },
+    {
+      "start": 5630.35,
+      "duration": 0.0,
+      "text": "minutes I might talk about systems or"
+    },
+    {
+      "start": 5630.36,
+      "duration": 0.0,
+      "text": "minutes I might talk about systems or just<01:33:50.560><c> answer</c><01:33:50.920><c> questions</c><01:33:51.480><c> yes</c><01:33:52.239><c> can</c><01:33:52.440><c> you</c><01:33:52.840><c> um</c><01:33:54.080><c> go</c>"
+    },
+    {
+      "start": 5634.27,
+      "duration": 0.0,
+      "text": "just answer questions yes can you um go"
+    },
+    {
+      "start": 5634.28,
+      "duration": 0.0,
+      "text": "just answer questions yes can you um go back<01:33:54.440><c> to</c><01:33:54.600><c> your</c><01:33:55.080><c> post</c><01:33:55.360><c> training</c><01:33:55.639><c> in</c><01:33:55.760><c> terms</c><01:33:55.960><c> of</c>"
+    },
+    {
+      "start": 5636.07,
+      "duration": 0.0,
+      "text": "back to your post training in terms of"
+    },
+    {
+      "start": 5636.08,
+      "duration": 0.0,
+      "text": "back to your post training in terms of post<01:33:56.719><c> training</c><01:33:57.719><c> how</c><01:33:57.920><c> did</c><01:33:58.119><c> we</c><01:33:58.320><c> tune</c><01:33:58.679><c> those</c>"
+    },
+    {
+      "start": 5638.87,
+      "duration": 0.0,
+      "text": "post training how did we tune those"
+    },
+    {
+      "start": 5638.88,
+      "duration": 0.0,
+      "text": "post training how did we tune those parameters<01:33:59.520><c> using</c><01:33:59.920><c> the</c><01:34:00.520><c> small</c><01:34:01.119><c> body</c><01:34:01.480><c> of</c>"
+    },
+    {
+      "start": 5641.95,
+      "duration": 0.0,
+      "text": "parameters using the small body of"
+    },
+    {
+      "start": 5641.96,
+      "duration": 0.0,
+      "text": "parameters using the small body of fine-tuning<01:34:02.960><c> data</c><01:34:03.400><c> and</c><01:34:03.600><c> have</c><01:34:03.800><c> such</c><01:34:04.080><c> big</c>"
+    },
+    {
+      "start": 5644.35,
+      "duration": 0.0,
+      "text": "fine-tuning data and have such big"
+    },
+    {
+      "start": 5644.36,
+      "duration": 0.0,
+      "text": "fine-tuning data and have such big effect<01:34:04.639><c> on</c><01:34:04.760><c> the</c><01:34:04.920><c> model</c><01:34:05.440><c> you</c><01:34:05.560><c> mentioned</c>"
+    },
+    {
+      "start": 5645.91,
+      "duration": 0.0,
+      "text": "effect on the model you mentioned"
+    },
+    {
+      "start": 5645.92,
+      "duration": 0.0,
+      "text": "effect on the model you mentioned earlier<01:34:06.320><c> that</c><01:34:06.639><c> there's</c><01:34:06.840><c> a</c><01:34:07.000><c> different</c><01:34:07.239><c> set</c><01:34:07.440><c> of</c>"
+    },
+    {
+      "start": 5647.83,
+      "duration": 0.0,
+      "text": "earlier that there's a different set of"
+    },
+    {
+      "start": 5647.84,
+      "duration": 0.0,
+      "text": "earlier that there's a different set of hyperparameters<01:34:08.840><c> are</c><01:34:09.000><c> we</c><01:34:09.119><c> changing</c><01:34:09.800><c> just</c>"
+    },
+    {
+      "start": 5650.03,
+      "duration": 0.0,
+      "text": "hyperparameters are we changing just"
+    },
+    {
+      "start": 5650.04,
+      "duration": 0.0,
+      "text": "hyperparameters are we changing just some<01:34:10.199><c> of</c><01:34:10.360><c> the</c><01:34:10.480><c> weights</c><01:34:10.800><c> the</c><01:34:10.920><c> later</c><01:34:11.280><c> weights</c><01:34:11.600><c> or</c>"
+    },
+    {
+      "start": 5651.79,
+      "duration": 0.0,
+      "text": "some of the weights the later weights or"
+    },
+    {
+      "start": 5651.8,
+      "duration": 0.0,
+      "text": "some of the weights the later weights or all<01:34:12.080><c> the</c><01:34:12.239><c> weights</c><01:34:12.679><c> what's</c><01:34:12.920><c> actually</c>"
+    },
+    {
+      "start": 5653.31,
+      "duration": 0.0,
+      "text": "all the weights what's actually"
+    },
+    {
+      "start": 5653.32,
+      "duration": 0.0,
+      "text": "all the weights what's actually happening<01:34:14.080><c> yeah</c><01:34:14.679><c> uh</c><01:34:14.880><c> yeah</c><01:34:15.040><c> I</c><01:34:15.199><c> I</c><01:34:15.280><c> kind</c><01:34:15.400><c> of</c>"
+    },
+    {
+      "start": 5655.51,
+      "duration": 0.0,
+      "text": "happening yeah uh yeah I I kind of"
+    },
+    {
+      "start": 5655.52,
+      "duration": 0.0,
+      "text": "happening yeah uh yeah I I kind of skimmed<01:34:15.840><c> through</c><01:34:16.000><c> all</c><01:34:16.080><c> of</c><01:34:16.239><c> this</c><01:34:16.560><c> you</c><01:34:16.719><c> change</c>"
+    },
+    {
+      "start": 5656.99,
+      "duration": 0.0,
+      "text": "skimmed through all of this you change"
+    },
+    {
+      "start": 5657.0,
+      "duration": 0.0,
+      "text": "skimmed through all of this you change all<01:34:17.119><c> the</c><01:34:17.239><c> weights</c><01:34:17.920><c> actually</c><01:34:18.560><c> um</c><01:34:19.000><c> industry</c>"
+    },
+    {
+      "start": 5659.43,
+      "duration": 0.0,
+      "text": "all the weights actually um industry"
+    },
+    {
+      "start": 5659.44,
+      "duration": 0.0,
+      "text": "all the weights actually um industry would<01:34:19.639><c> change</c><01:34:19.880><c> all</c><01:34:20.000><c> the</c><01:34:20.159><c> weights</c><01:34:20.679><c> in</c><01:34:20.920><c> open</c>"
+    },
+    {
+      "start": 5661.229,
+      "duration": 0.0,
+      "text": "would change all the weights in open"
+    },
+    {
+      "start": 5661.239,
+      "duration": 0.0,
+      "text": "would change all the weights in open source<01:34:21.520><c> land</c><01:34:22.000><c> you</c><01:34:22.119><c> might</c><01:34:22.360><c> have</c><01:34:22.639><c> heard</c><01:34:22.920><c> of</c>"
+    },
+    {
+      "start": 5663.109,
+      "duration": 0.0,
+      "text": "source land you might have heard of"
+    },
+    {
+      "start": 5663.119,
+      "duration": 0.0,
+      "text": "source land you might have heard of Laura<01:34:23.920><c> which</c><01:34:24.080><c> is</c><01:34:24.239><c> going</c><01:34:24.360><c> to</c><01:34:24.880><c> change</c><01:34:25.880><c> basically</c>"
+    },
+    {
+      "start": 5666.43,
+      "duration": 0.0,
+      "text": "Laura which is going to change basically"
+    },
+    {
+      "start": 5666.44,
+      "duration": 0.0,
+      "text": "Laura which is going to change basically only<01:34:26.679><c> some</c><01:34:26.960><c> of</c><01:34:27.080><c> the</c><01:34:27.199><c> weights</c><01:34:27.719><c> or</c><01:34:27.920><c> it</c><01:34:28.080><c> actually</c>"
+    },
+    {
+      "start": 5668.47,
+      "duration": 0.0,
+      "text": "only some of the weights or it actually"
+    },
+    {
+      "start": 5668.48,
+      "duration": 0.0,
+      "text": "only some of the weights or it actually to<01:34:28.600><c> be</c><01:34:28.760><c> more</c><01:34:29.000><c> specific</c><01:34:29.639><c> it's</c><01:34:29.800><c> going</c><01:34:29.880><c> to</c><01:34:30.159><c> add</c>"
+    },
+    {
+      "start": 5670.43,
+      "duration": 0.0,
+      "text": "to be more specific it's going to add"
+    },
+    {
+      "start": 5670.44,
+      "duration": 0.0,
+      "text": "to be more specific it's going to add some<01:34:30.719><c> differences</c><01:34:31.159><c> to</c><01:34:31.320><c> the</c><01:34:31.440><c> output</c><01:34:31.840><c> of</c><01:34:32.040><c> every</c>"
+    },
+    {
+      "start": 5672.39,
+      "duration": 0.0,
+      "text": "some differences to the output of every"
+    },
+    {
+      "start": 5672.4,
+      "duration": 0.0,
+      "text": "some differences to the output of every of<01:34:32.560><c> every</c><01:34:32.760><c> layer</c><01:34:33.360><c> but</c><01:34:33.560><c> but</c><01:34:33.639><c> in</c><01:34:33.800><c> Industry</c>"
+    },
+    {
+      "start": 5674.149,
+      "duration": 0.0,
+      "text": "of every layer but but in Industry"
+    },
+    {
+      "start": 5674.159,
+      "duration": 0.0,
+      "text": "of every layer but but in Industry you're<01:34:34.239><c> going</c><01:34:34.360><c> to</c><01:34:34.520><c> just</c><01:34:34.719><c> fine</c><01:34:34.960><c> tune</c><01:34:35.199><c> all</c><01:34:35.400><c> the</c>"
+    },
+    {
+      "start": 5675.99,
+      "duration": 0.0,
+      "text": "you're going to just fine tune all the"
+    },
+    {
+      "start": 5676.0,
+      "duration": 0.0,
+      "text": "you're going to just fine tune all the weights<01:34:37.000><c> um</c><01:34:37.840><c> and</c><01:34:38.840><c> also</c><01:34:39.080><c> to</c><01:34:39.199><c> say</c><01:34:39.360><c> something</c>"
+    },
+    {
+      "start": 5679.59,
+      "duration": 0.0,
+      "text": "weights um and also to say something"
+    },
+    {
+      "start": 5679.6,
+      "duration": 0.0,
+      "text": "weights um and also to say something else<01:34:39.800><c> about</c><01:34:39.960><c> the</c><01:34:40.119><c> data</c><01:34:40.480><c> actually</c><01:34:40.719><c> the</c><01:34:40.880><c> SL</c><01:34:41.239><c> St</c>"
+    },
+    {
+      "start": 5681.39,
+      "duration": 0.0,
+      "text": "else about the data actually the SL St"
+    },
+    {
+      "start": 5681.4,
+      "duration": 0.0,
+      "text": "else about the data actually the SL St all<01:34:41.600><c> HF</c><01:34:42.119><c> you</c><01:34:42.280><c> usually</c><01:34:42.560><c> going</c><01:34:42.679><c> to</c><01:34:42.800><c> collect</c><01:34:43.800><c> uh</c><01:34:44.040><c> a</c>"
+    },
+    {
+      "start": 5684.189,
+      "duration": 0.0,
+      "text": "all HF you usually going to collect uh a"
+    },
+    {
+      "start": 5684.199,
+      "duration": 0.0,
+      "text": "all HF you usually going to collect uh a lot<01:34:44.400><c> more</c><01:34:44.600><c> data</c><01:34:44.880><c> than</c><01:34:45.040><c> with</c><01:34:45.199><c> sft</c><01:34:45.679><c> so</c><01:34:45.840><c> if</c><01:34:46.000><c> fft</c><01:34:46.600><c> is</c>"
+    },
+    {
+      "start": 5686.79,
+      "duration": 0.0,
+      "text": "lot more data than with sft so if fft is"
+    },
+    {
+      "start": 5686.8,
+      "duration": 0.0,
+      "text": "lot more data than with sft so if fft is like<01:34:47.159><c> 5,000</c><01:34:48.159><c> 10,000</c><01:34:48.840><c> maybe</c><01:34:49.080><c> 50,000</c><01:34:50.080><c> with</c><01:34:50.360><c> rhf</c>"
+    },
+    {
+      "start": 5691.189,
+      "duration": 0.0,
+      "text": "like 5,000 10,000 maybe 50,000 with rhf"
+    },
+    {
+      "start": 5691.199,
+      "duration": 0.0,
+      "text": "like 5,000 10,000 maybe 50,000 with rhf I<01:34:51.320><c> think</c><01:34:51.480><c> you're</c><01:34:51.639><c> going</c><01:34:51.719><c> to</c><01:34:52.080><c> be</c><01:34:52.239><c> more</c><01:34:52.480><c> around</c>"
+    },
+    {
+      "start": 5692.669,
+      "duration": 0.0,
+      "text": "I think you're going to be more around"
+    },
+    {
+      "start": 5692.679,
+      "duration": 0.0,
+      "text": "I think you're going to be more around like<01:34:52.840><c> the</c><01:34:53.000><c> 1</c><01:34:53.159><c> million</c>"
+    },
+    {
+      "start": 5694.31,
+      "duration": 0.0,
+      "text": "like the 1 million"
+    },
+    {
+      "start": 5694.32,
+      "duration": 0.0,
+      "text": "like the 1 million uh<01:34:54.440><c> order</c><01:34:54.719><c> of</c><01:34:54.880><c> magnitude</c><01:34:55.360><c> it's</c><01:34:55.480><c> still</c><01:34:55.719><c> much</c>"
+    },
+    {
+      "start": 5695.95,
+      "duration": 0.0,
+      "text": "uh order of magnitude it's still much"
+    },
+    {
+      "start": 5695.96,
+      "duration": 0.0,
+      "text": "uh order of magnitude it's still much less<01:34:56.159><c> than</c><01:34:56.320><c> pre-training</c><01:34:56.920><c> though</c><01:34:57.520><c> yeah</c>"
+    },
+    {
+      "start": 5697.87,
+      "duration": 0.0,
+      "text": "less than pre-training though yeah"
+    },
+    {
+      "start": 5697.88,
+      "duration": 0.0,
+      "text": "less than pre-training though yeah because<01:34:58.199><c> pre-training</c><01:34:58.639><c> is</c><01:34:58.800><c> 15</c><01:34:59.199><c> trillion</c>"
+    },
+    {
+      "start": 5699.669,
+      "duration": 0.0,
+      "text": "because pre-training is 15 trillion"
+    },
+    {
+      "start": 5699.679,
+      "duration": 0.0,
+      "text": "because pre-training is 15 trillion tokens<01:35:00.239><c> I</c><01:35:00.320><c> mean</c><01:35:00.520><c> this</c><01:35:00.639><c> is</c><01:35:01.000><c> like</c><01:35:01.480><c> that's</c><01:35:01.719><c> not</c>"
+    },
+    {
+      "start": 5701.83,
+      "duration": 0.0,
+      "text": "tokens I mean this is like that's not"
+    },
+    {
+      "start": 5701.84,
+      "duration": 0.0,
+      "text": "tokens I mean this is like that's not even<01:35:02.040><c> a</c><01:35:02.159><c> drop</c><01:35:02.679><c> and</c><01:35:02.920><c> yet</c><01:35:03.239><c> you</c><01:35:03.520><c> influence</c><01:35:04.000><c> the</c>"
+    },
+    {
+      "start": 5704.109,
+      "duration": 0.0,
+      "text": "even a drop and yet you influence the"
+    },
+    {
+      "start": 5704.119,
+      "duration": 0.0,
+      "text": "even a drop and yet you influence the weight<01:35:04.440><c> a</c><01:35:04.600><c> lot</c><01:35:05.000><c> so</c><01:35:05.360><c> because</c><01:35:05.520><c> you</c><01:35:05.679><c> do</c><01:35:05.800><c> it</c><01:35:05.960><c> I</c><01:35:06.000><c> mean</c>"
+    },
+    {
+      "start": 5706.109,
+      "duration": 0.0,
+      "text": "weight a lot so because you do it I mean"
+    },
+    {
+      "start": 5706.119,
+      "duration": 0.0,
+      "text": "weight a lot so because you do it I mean you<01:35:06.239><c> have</c><01:35:06.360><c> to</c><01:35:06.520><c> think</c><01:35:06.719><c> that</c><01:35:06.920><c> how</c><01:35:07.080><c> you</c><01:35:07.199><c> do</c><01:35:07.320><c> it</c><01:35:07.920><c> is</c>"
+    },
+    {
+      "start": 5708.43,
+      "duration": 0.0,
+      "text": "you have to think that how you do it is"
+    },
+    {
+      "start": 5708.44,
+      "duration": 0.0,
+      "text": "you have to think that how you do it is you<01:35:08.920><c> use</c><01:35:09.920><c> um</c><01:35:10.679><c> I</c><01:35:10.760><c> mean</c><01:35:11.080><c> as</c><01:35:11.199><c> I</c><01:35:11.320><c> said</c><01:35:11.560><c> the</c><01:35:12.000><c> learning</c>"
+    },
+    {
+      "start": 5712.27,
+      "duration": 0.0,
+      "text": "you use um I mean as I said the learning"
+    },
+    {
+      "start": 5712.28,
+      "duration": 0.0,
+      "text": "you use um I mean as I said the learning rate<01:35:12.480><c> that</c><01:35:12.560><c> you're</c><01:35:12.679><c> going</c><01:35:12.760><c> to</c><01:35:12.840><c> use</c><01:35:13.000><c> is</c><01:35:13.119><c> going</c>"
+    },
+    {
+      "start": 5713.189,
+      "duration": 0.0,
+      "text": "rate that you're going to use is going"
+    },
+    {
+      "start": 5713.199,
+      "duration": 0.0,
+      "text": "rate that you're going to use is going to<01:35:13.280><c> be</c><01:35:13.400><c> different</c><01:35:14.119><c> but</c><01:35:14.320><c> also</c><01:35:15.159><c> you</c><01:35:15.360><c> only</c><01:35:15.679><c> do</c>"
+    },
+    {
+      "start": 5715.95,
+      "duration": 0.0,
+      "text": "to be different but also you only do"
+    },
+    {
+      "start": 5715.96,
+      "duration": 0.0,
+      "text": "to be different but also you only do that<01:35:16.199><c> so</c><01:35:16.679><c> just</c><01:35:16.880><c> imagine</c><01:35:17.199><c> if</c><01:35:17.320><c> I</c><01:35:17.440><c> train</c><01:35:18.119><c> even</c><01:35:18.320><c> if</c>"
+    },
+    {
+      "start": 5718.39,
+      "duration": 0.0,
+      "text": "that so just imagine if I train even if"
+    },
+    {
+      "start": 5718.4,
+      "duration": 0.0,
+      "text": "that so just imagine if I train even if I<01:35:18.520><c> train</c><01:35:18.760><c> on</c><01:35:18.920><c> one</c><01:35:19.119><c> sentence</c><01:35:20.119><c> but</c><01:35:20.400><c> over</c><01:35:20.600><c> and</c>"
+    },
+    {
+      "start": 5720.79,
+      "duration": 0.0,
+      "text": "I train on one sentence but over and"
+    },
+    {
+      "start": 5720.8,
+      "duration": 0.0,
+      "text": "I train on one sentence but over and over<01:35:21.080><c> again</c><01:35:21.679><c> all</c><01:35:22.159><c> at</c><01:35:22.320><c> some</c><01:35:22.480><c> point</c><01:35:22.719><c> my</c><01:35:22.840><c> model</c>"
+    },
+    {
+      "start": 5723.07,
+      "duration": 0.0,
+      "text": "over again all at some point my model"
+    },
+    {
+      "start": 5723.08,
+      "duration": 0.0,
+      "text": "over again all at some point my model will<01:35:23.239><c> only</c><01:35:23.960><c> that</c><01:35:24.080><c> sentence</c><01:35:24.679><c> even</c><01:35:25.000><c> if</c><01:35:25.960><c> uh</c><01:35:26.199><c> it</c>"
+    },
+    {
+      "start": 5726.31,
+      "duration": 0.0,
+      "text": "will only that sentence even if uh it"
+    },
+    {
+      "start": 5726.32,
+      "duration": 0.0,
+      "text": "will only that sentence even if uh it was<01:35:26.520><c> just</c><01:35:26.679><c> one</c><01:35:26.880><c> sentence</c><01:35:27.360><c> instead</c><01:35:27.639><c> of</c><01:35:27.760><c> the</c><01:35:27.880><c> 15</c>"
+    },
+    {
+      "start": 5728.149,
+      "duration": 0.0,
+      "text": "was just one sentence instead of the 15"
+    },
+    {
+      "start": 5728.159,
+      "duration": 0.0,
+      "text": "was just one sentence instead of the 15 trillion<01:35:28.520><c> tokens</c><01:35:29.159><c> so</c><01:35:29.320><c> if</c><01:35:29.440><c> you</c><01:35:29.600><c> use</c><01:35:29.840><c> a</c><01:35:30.040><c> large</c>"
+    },
+    {
+      "start": 5730.31,
+      "duration": 0.0,
+      "text": "trillion tokens so if you use a large"
+    },
+    {
+      "start": 5730.32,
+      "duration": 0.0,
+      "text": "trillion tokens so if you use a large enough<01:35:30.600><c> learning</c><01:35:30.920><c> rate</c><01:35:31.400><c> and</c><01:35:31.520><c> for</c><01:35:32.040><c> enough</c><01:35:32.400><c> time</c>"
+    },
+    {
+      "start": 5732.87,
+      "duration": 0.0,
+      "text": "enough learning rate and for enough time"
+    },
+    {
+      "start": 5732.88,
+      "duration": 0.0,
+      "text": "enough learning rate and for enough time you<01:35:33.000><c> will</c><01:35:33.239><c> basically</c><01:35:33.800><c> overfit</c><01:35:34.320><c> that</c><01:35:34.480><c> sentence</c>"
+    },
+    {
+      "start": 5735.149,
+      "duration": 0.0,
+      "text": "you will basically overfit that sentence"
+    },
+    {
+      "start": 5735.159,
+      "duration": 0.0,
+      "text": "you will basically overfit that sentence so<01:35:35.400><c> the</c><01:35:35.600><c> the</c><01:35:35.760><c> the</c><01:35:35.920><c> key</c><01:35:36.119><c> thing</c><01:35:36.280><c> to</c><01:35:36.560><c> to</c><01:35:36.760><c> remember</c>"
+    },
+    {
+      "start": 5737.149,
+      "duration": 0.0,
+      "text": "so the the the key thing to to remember"
+    },
+    {
+      "start": 5737.159,
+      "duration": 0.0,
+      "text": "so the the the key thing to to remember is<01:35:37.360><c> that</c><01:35:38.159><c> um</c><01:35:38.760><c> the</c><01:35:38.920><c> data</c><01:35:39.159><c> is</c><01:35:39.280><c> not</c><01:35:39.440><c> I</c><01:35:39.760><c> it's</c><01:35:39.880><c> not</c><01:35:40.040><c> as</c>"
+    },
+    {
+      "start": 5740.149,
+      "duration": 0.0,
+      "text": "is that um the data is not I it's not as"
+    },
+    {
+      "start": 5740.159,
+      "duration": 0.0,
+      "text": "is that um the data is not I it's not as if<01:35:40.320><c> you</c><01:35:40.520><c> mix</c><01:35:41.280><c> some</c><01:35:41.520><c> posttraining</c><01:35:41.960><c> data</c><01:35:42.560><c> and</c>"
+    },
+    {
+      "start": 5742.669,
+      "duration": 0.0,
+      "text": "if you mix some posttraining data and"
+    },
+    {
+      "start": 5742.679,
+      "duration": 0.0,
+      "text": "if you mix some posttraining data and some<01:35:42.880><c> pre-training</c><01:35:43.400><c> data</c><01:35:43.800><c> you</c><01:35:43.920><c> do</c>"
+    },
+    {
+      "start": 5744.109,
+      "duration": 0.0,
+      "text": "some pre-training data you do"
+    },
+    {
+      "start": 5744.119,
+      "duration": 0.0,
+      "text": "some pre-training data you do pre-training<01:35:44.960><c> and</c><01:35:45.119><c> then</c><01:35:45.280><c> you</c><01:35:45.719><c> just</c><01:35:45.880><c> start</c>"
+    },
+    {
+      "start": 5746.149,
+      "duration": 0.0,
+      "text": "pre-training and then you just start"
+    },
+    {
+      "start": 5746.159,
+      "duration": 0.0,
+      "text": "pre-training and then you just start fine-tuning<01:35:47.040><c> only</c><01:35:47.239><c> on</c><01:35:47.360><c> the</c><01:35:47.480><c> post</c><01:35:47.719><c> trining</c><01:35:48.119><c> so</c>"
+    },
+    {
+      "start": 5748.31,
+      "duration": 0.0,
+      "text": "fine-tuning only on the post trining so"
+    },
+    {
+      "start": 5748.32,
+      "duration": 0.0,
+      "text": "fine-tuning only on the post trining so another<01:35:48.600><c> way</c><01:35:49.119><c> maybe</c><01:35:49.360><c> another</c><01:35:49.639><c> perspective</c><01:35:50.400><c> is</c>"
+    },
+    {
+      "start": 5750.51,
+      "duration": 0.0,
+      "text": "another way maybe another perspective is"
+    },
+    {
+      "start": 5750.52,
+      "duration": 0.0,
+      "text": "another way maybe another perspective is that<01:35:50.679><c> the</c><01:35:50.880><c> post</c><01:35:51.119><c> the</c><01:35:51.239><c> pre-training</c><01:35:52.080><c> is</c><01:35:52.199><c> just</c>"
+    },
+    {
+      "start": 5752.35,
+      "duration": 0.0,
+      "text": "that the post the pre-training is just"
+    },
+    {
+      "start": 5752.36,
+      "duration": 0.0,
+      "text": "that the post the pre-training is just the<01:35:52.560><c> initialization</c><01:35:53.159><c> of</c><01:35:53.280><c> your</c><01:35:53.400><c> model</c>"
+    },
+    {
+      "start": 5754.149,
+      "duration": 0.0,
+      "text": "the initialization of your model"
+    },
+    {
+      "start": 5754.159,
+      "duration": 0.0,
+      "text": "the initialization of your model and<01:35:54.280><c> once</c><01:35:54.440><c> you</c><01:35:54.600><c> view</c><01:35:54.800><c> it</c><01:35:55.000><c> that</c><01:35:55.199><c> way</c><01:35:55.520><c> that</c><01:35:55.719><c> this</c>"
+    },
+    {
+      "start": 5755.79,
+      "duration": 0.0,
+      "text": "and once you view it that way that this"
+    },
+    {
+      "start": 5755.8,
+      "duration": 0.0,
+      "text": "and once you view it that way that this is<01:35:56.000><c> just</c><01:35:56.199><c> initialization</c><01:35:56.800><c> of</c><01:35:57.080><c> Weights</c><01:35:58.080><c> then</c>"
+    },
+    {
+      "start": 5758.189,
+      "duration": 0.0,
+      "text": "is just initialization of Weights then"
+    },
+    {
+      "start": 5758.199,
+      "duration": 0.0,
+      "text": "is just initialization of Weights then there's<01:35:58.440><c> nothing</c><01:35:58.800><c> special</c><01:35:59.639><c> like</c><01:35:59.880><c> you</c><01:36:00.000><c> don't</c>"
+    },
+    {
+      "start": 5760.189,
+      "duration": 0.0,
+      "text": "there's nothing special like you don't"
+    },
+    {
+      "start": 5760.199,
+      "duration": 0.0,
+      "text": "there's nothing special like you don't need<01:36:00.360><c> to</c><01:36:00.600><c> remember</c><01:36:00.960><c> that</c><01:36:01.040><c> you</c><01:36:01.119><c> train</c><01:36:01.440><c> a</c><01:36:01.600><c> lot</c><01:36:01.719><c> of</c>"
+    },
+    {
+      "start": 5761.83,
+      "duration": 0.0,
+      "text": "need to remember that you train a lot of"
+    },
+    {
+      "start": 5761.84,
+      "duration": 0.0,
+      "text": "need to remember that you train a lot of data<01:36:02.159><c> before</c><01:36:02.639><c> the</c><01:36:02.760><c> only</c><01:36:02.920><c> thing</c><01:36:03.040><c> that</c><01:36:03.159><c> matters</c>"
+    },
+    {
+      "start": 5763.47,
+      "duration": 0.0,
+      "text": "data before the only thing that matters"
+    },
+    {
+      "start": 5763.48,
+      "duration": 0.0,
+      "text": "data before the only thing that matters is<01:36:03.600><c> that</c><01:36:03.679><c> you</c><01:36:03.760><c> had</c><01:36:03.880><c> an</c><01:36:04.000><c> initialization</c><01:36:05.000><c> and</c>"
+    },
+    {
+      "start": 5765.109,
+      "duration": 0.0,
+      "text": "is that you had an initialization and"
+    },
+    {
+      "start": 5765.119,
+      "duration": 0.0,
+      "text": "is that you had an initialization and now<01:36:05.280><c> I</c><01:36:05.440><c> actually</c><01:36:05.600><c> train</c><01:36:05.840><c> a</c><01:36:06.000><c> model</c><01:36:06.480><c> so</c><01:36:06.639><c> maybe</c>"
+    },
+    {
+      "start": 5766.87,
+      "duration": 0.0,
+      "text": "now I actually train a model so maybe"
+    },
+    {
+      "start": 5766.88,
+      "duration": 0.0,
+      "text": "now I actually train a model so maybe think<01:36:07.000><c> about</c><01:36:07.159><c> it</c><01:36:07.360><c> that</c><01:36:07.520><c> way</c><01:36:07.880><c> like</c><01:36:08.080><c> there's</c><01:36:08.360><c> a</c>"
+    },
+    {
+      "start": 5768.669,
+      "duration": 0.0,
+      "text": "think about it that way like there's a"
+    },
+    {
+      "start": 5768.679,
+      "duration": 0.0,
+      "text": "think about it that way like there's a there's<01:36:08.800><c> a</c><01:36:08.920><c> mark</c><01:36:09.119><c> of</c><01:36:09.280><c> property</c><01:36:09.920><c> in</c><01:36:10.119><c> some</c><01:36:10.360><c> way</c>"
+    },
+    {
+      "start": 5770.629,
+      "duration": 0.0,
+      "text": "there's a mark of property in some way"
+    },
+    {
+      "start": 5770.639,
+      "duration": 0.0,
+      "text": "there's a mark of property in some way just<01:36:10.760><c> like</c><01:36:10.880><c> you</c><01:36:11.000><c> had</c><01:36:11.159><c> your</c><01:36:11.320><c> weights</c><01:36:11.719><c> this</c><01:36:11.800><c> is</c>"
+    },
+    {
+      "start": 5771.91,
+      "duration": 0.0,
+      "text": "just like you had your weights this is"
+    },
+    {
+      "start": 5771.92,
+      "duration": 0.0,
+      "text": "just like you had your weights this is my<01:36:12.080><c> initialization</c><01:36:12.960><c> now</c><01:36:13.080><c> I'm</c><01:36:13.199><c> training</c><01:36:13.560><c> that</c>"
+    },
+    {
+      "start": 5773.709,
+      "duration": 0.0,
+      "text": "my initialization now I'm training that"
+    },
+    {
+      "start": 5773.719,
+      "duration": 0.0,
+      "text": "my initialization now I'm training that one<01:36:14.400><c> does</c><01:36:14.600><c> that</c><01:36:14.800><c> kind</c><01:36:14.880><c> of</c><01:36:15.040><c> answer</c><01:36:15.280><c> your</c>"
+    },
+    {
+      "start": 5775.43,
+      "duration": 0.0,
+      "text": "one does that kind of answer your"
+    },
+    {
+      "start": 5775.44,
+      "duration": 0.0,
+      "text": "one does that kind of answer your question<01:36:16.400><c> kind</c><01:36:16.560><c> of</c><01:36:17.400><c> but</c><01:36:18.400><c> you</c><01:36:18.520><c> said</c><01:36:18.760><c> something</c>"
+    },
+    {
+      "start": 5779.109,
+      "duration": 0.0,
+      "text": "question kind of but you said something"
+    },
+    {
+      "start": 5779.119,
+      "duration": 0.0,
+      "text": "question kind of but you said something just<01:36:19.280><c> now</c><01:36:19.639><c> about</c><01:36:20.480><c> it's</c><01:36:20.719><c> almost</c><01:36:21.000><c> the</c>"
+    },
+    {
+      "start": 5781.149,
+      "duration": 0.0,
+      "text": "just now about it's almost the"
+    },
+    {
+      "start": 5781.159,
+      "duration": 0.0,
+      "text": "just now about it's almost the equivalence<01:36:21.840><c> of</c><01:36:22.119><c> just</c><01:36:22.360><c> rerunning</c><01:36:22.920><c> the</c><01:36:23.280><c> find</c>"
+    },
+    {
+      "start": 5783.59,
+      "duration": 0.0,
+      "text": "equivalence of just rerunning the find"
+    },
+    {
+      "start": 5783.6,
+      "duration": 0.0,
+      "text": "equivalence of just rerunning the find tuning<01:36:23.880><c> data</c><01:36:24.199><c> many</c><01:36:24.520><c> times</c><01:36:25.239><c> is</c><01:36:25.320><c> it</c><01:36:25.560><c> actually</c><01:36:26.119><c> is</c>"
+    },
+    {
+      "start": 5786.229,
+      "duration": 0.0,
+      "text": "tuning data many times is it actually is"
+    },
+    {
+      "start": 5786.239,
+      "duration": 0.0,
+      "text": "tuning data many times is it actually is that<01:36:26.560><c> what</c><01:36:26.800><c> actually</c><01:36:27.159><c> happens</c><01:36:27.560><c> in</c><01:36:27.679><c> order</c><01:36:28.080><c> to</c>"
+    },
+    {
+      "start": 5789.07,
+      "duration": 0.0,
+      "text": "that what actually happens in order to"
+    },
+    {
+      "start": 5789.08,
+      "duration": 0.0,
+      "text": "that what actually happens in order to give<01:36:29.280><c> so</c><01:36:29.440><c> much</c><01:36:29.600><c> more</c><01:36:30.159><c> preference</c>"
+    },
+    {
+      "start": 5792.83,
+      "duration": 0.0,
+      "text": "give so much more preference"
+    },
+    {
+      "start": 5792.84,
+      "duration": 0.0,
+      "text": "give so much more preference um<01:36:33.840><c> you</c><01:36:34.239><c> might</c><01:36:34.920><c> I</c><01:36:35.080><c> actually</c><01:36:35.400><c> don't</c><01:36:35.679><c> know</c><01:36:36.000><c> right</c>"
+    },
+    {
+      "start": 5796.189,
+      "duration": 0.0,
+      "text": "um you might I actually don't know right"
+    },
+    {
+      "start": 5796.199,
+      "duration": 0.0,
+      "text": "um you might I actually don't know right now<01:36:36.360><c> how</c><01:36:36.520><c> they</c><01:36:36.639><c> do</c><01:36:36.800><c> it</c><01:36:36.880><c> in</c><01:36:37.040><c> Industry</c><01:36:37.719><c> when</c><01:36:37.960><c> we</c>"
+    },
+    {
+      "start": 5798.109,
+      "duration": 0.0,
+      "text": "now how they do it in Industry when we"
+    },
+    {
+      "start": 5798.119,
+      "duration": 0.0,
+      "text": "now how they do it in Industry when we did<01:36:38.320><c> alpaca</c><01:36:38.840><c> we</c><01:36:38.960><c> had</c><01:36:39.080><c> to</c><01:36:39.159><c> do</c><01:36:39.280><c> three</c><01:36:39.760><c> box</c><01:36:40.080><c> so</c><01:36:40.239><c> you</c>"
+    },
+    {
+      "start": 5800.35,
+      "duration": 0.0,
+      "text": "did alpaca we had to do three box so you"
+    },
+    {
+      "start": 5800.36,
+      "duration": 0.0,
+      "text": "did alpaca we had to do three box so you did<01:36:40.760><c> run</c><01:36:40.960><c> it</c><01:36:41.159><c> three</c><01:36:41.400><c> times</c><01:36:41.679><c> to</c><01:36:41.920><c> it</c>"
+    },
+    {
+      "start": 5803.83,
+      "duration": 0.0,
+      "text": "did run it three times to it"
+    },
+    {
+      "start": 5803.84,
+      "duration": 0.0,
+      "text": "did run it three times to it um<01:36:44.840><c> but</c><01:36:45.280><c> I</c><01:36:45.320><c> mean</c><01:36:45.520><c> even</c><01:36:45.679><c> the</c><01:36:45.840><c> number</c><01:36:46.000><c> of</c><01:36:46.159><c> times</c>"
+    },
+    {
+      "start": 5806.39,
+      "duration": 0.0,
+      "text": "um but I mean even the number of times"
+    },
+    {
+      "start": 5806.4,
+      "duration": 0.0,
+      "text": "um but I mean even the number of times that<01:36:46.560><c> you</c><01:36:46.679><c> run</c><01:36:46.920><c> it</c><01:36:47.080><c> through</c><01:36:47.360><c> it's</c><01:36:47.560><c> actually</c>"
+    },
+    {
+      "start": 5807.79,
+      "duration": 0.0,
+      "text": "that you run it through it's actually"
+    },
+    {
+      "start": 5807.8,
+      "duration": 0.0,
+      "text": "that you run it through it's actually not<01:36:48.040><c> important</c><01:36:48.560><c> the</c><01:36:48.679><c> only</c><01:36:48.920><c> thing</c><01:36:49.320><c> like</c><01:36:49.800><c> the</c>"
+    },
+    {
+      "start": 5809.91,
+      "duration": 0.0,
+      "text": "not important the only thing like the"
+    },
+    {
+      "start": 5809.92,
+      "duration": 0.0,
+      "text": "not important the only thing like the only<01:36:50.280><c> thing</c><01:36:50.440><c> is</c><01:36:50.600><c> the</c><01:36:51.040><c> is</c><01:36:51.159><c> kind</c><01:36:51.280><c> of</c><01:36:51.400><c> the</c>"
+    },
+    {
+      "start": 5811.55,
+      "duration": 0.0,
+      "text": "only thing is the is kind of the"
+    },
+    {
+      "start": 5811.56,
+      "duration": 0.0,
+      "text": "only thing is the is kind of the effective<01:36:51.840><c> learning</c><01:36:52.199><c> rate</c><01:36:52.639><c> that</c><01:36:52.880><c> what</c>"
+    },
+    {
+      "start": 5813.03,
+      "duration": 0.0,
+      "text": "effective learning rate that what"
+    },
+    {
+      "start": 5813.04,
+      "duration": 0.0,
+      "text": "effective learning rate that what matters"
+    },
+    {
+      "start": 5814.149,
+      "duration": 0.0,
+      "text": "matters"
+    },
+    {
+      "start": 5814.159,
+      "duration": 0.0,
+      "text": "matters um<01:36:54.880><c> so</c>"
+    },
+    {
+      "start": 5816.109,
+      "duration": 0.0,
+      "text": "um so"
+    },
+    {
+      "start": 5816.119,
+      "duration": 0.0,
+      "text": "um so yeah"
+    },
+    {
+      "start": 5817.79,
+      "duration": 0.0,
+      "text": "yeah"
+    },
+    {
+      "start": 5817.8,
+      "duration": 0.0,
+      "text": "yeah great<01:36:58.800><c> so</c><01:36:59.440><c> I</c><01:36:59.520><c> think</c><01:36:59.800><c> I</c><01:36:59.960><c> have</c><01:37:00.239><c> five</c><01:37:00.440><c> minutes</c>"
+    },
+    {
+      "start": 5822.96,
+      "duration": 0.0,
+      "text": "[Music]"
+    },
+    {
+      "start": 5825.31,
+      "duration": 0.0,
+      "text": "[Music]"
+    },
+    {
+      "start": 5825.32,
+      "duration": 0.0,
+      "text": "[Music] right<01:37:06.320><c> okay</c><01:37:07.080><c> I</c><01:37:08.080><c> might</c><01:37:08.920><c> try</c><01:37:09.560><c> to</c><01:37:09.920><c> give</c><01:37:10.080><c> a</c><01:37:10.320><c> high</c>"
+    },
+    {
+      "start": 5830.55,
+      "duration": 0.0,
+      "text": "right okay I might try to give a high"
+    },
+    {
+      "start": 5830.56,
+      "duration": 0.0,
+      "text": "right okay I might try to give a high level<01:37:10.920><c> Overview</c><01:37:11.800><c> at</c><01:37:11.920><c> least</c><01:37:12.119><c> from</c><01:37:12.400><c> one</c><01:37:12.600><c> of</c><01:37:12.760><c> the</c>"
+    },
+    {
+      "start": 5832.95,
+      "duration": 0.0,
+      "text": "level Overview at least from one of the"
+    },
+    {
+      "start": 5832.96,
+      "duration": 0.0,
+      "text": "level Overview at least from one of the systems<01:37:13.880><c> trick</c><01:37:14.880><c> systems</c><01:37:15.719><c> as</c><01:37:15.880><c> we</c><01:37:16.040><c> said</c><01:37:17.000><c> uh</c><01:37:17.320><c> for</c>"
+    },
+    {
+      "start": 5837.669,
+      "duration": 0.0,
+      "text": "systems trick systems as we said uh for"
+    },
+    {
+      "start": 5837.679,
+      "duration": 0.0,
+      "text": "systems trick systems as we said uh for everyone<01:37:18.119><c> Bott</c><01:37:18.440><c> neck</c><01:37:18.679><c> is</c><01:37:18.760><c> a</c><01:37:19.239><c> sorry</c><01:37:19.520><c> compute</c><01:37:19.880><c> is</c>"
+    },
+    {
+      "start": 5839.99,
+      "duration": 0.0,
+      "text": "everyone Bott neck is a sorry compute is"
+    },
+    {
+      "start": 5840.0,
+      "duration": 0.0,
+      "text": "everyone Bott neck is a sorry compute is the<01:37:20.159><c> huge</c><01:37:20.560><c> bottleneck</c><01:37:21.560><c> uh</c><01:37:21.679><c> one</c><01:37:21.880><c> question</c><01:37:22.080><c> you</c>"
+    },
+    {
+      "start": 5842.189,
+      "duration": 0.0,
+      "text": "the huge bottleneck uh one question you"
+    },
+    {
+      "start": 5842.199,
+      "duration": 0.0,
+      "text": "the huge bottleneck uh one question you might<01:37:22.400><c> ask</c><01:37:22.600><c> is</c><01:37:22.760><c> why</c><01:37:22.920><c> not</c><01:37:23.080><c> buy</c><01:37:23.239><c> more</c><01:37:23.760><c> gpus</c><01:37:24.760><c> uh</c>"
+    },
+    {
+      "start": 5844.99,
+      "duration": 0.0,
+      "text": "might ask is why not buy more gpus uh"
+    },
+    {
+      "start": 5845.0,
+      "duration": 0.0,
+      "text": "might ask is why not buy more gpus uh gpus<01:37:25.440><c> are</c><01:37:25.639><c> expensive</c><01:37:26.040><c> but</c><01:37:26.199><c> also</c><01:37:26.360><c> are</c><01:37:26.520><c> scarce</c>"
+    },
+    {
+      "start": 5846.91,
+      "duration": 0.0,
+      "text": "gpus are expensive but also are scarce"
+    },
+    {
+      "start": 5846.92,
+      "duration": 0.0,
+      "text": "gpus are expensive but also are scarce even<01:37:27.080><c> if</c><01:37:27.159><c> you</c><01:37:27.280><c> have</c><01:37:27.320><c> $10</c><01:37:27.560><c> million</c><01:37:28.159><c> right</c><01:37:28.280><c> now</c>"
+    },
+    {
+      "start": 5848.51,
+      "duration": 0.0,
+      "text": "even if you have $10 million right now"
+    },
+    {
+      "start": 5848.52,
+      "duration": 0.0,
+      "text": "even if you have $10 million right now you<01:37:28.679><c> cannot</c><01:37:29.040><c> buy</c><01:37:29.320><c> the</c><01:37:29.440><c> best</c><01:37:29.880><c> gpus</c><01:37:30.880><c> um</c>"
+    },
+    {
+      "start": 5852.27,
+      "duration": 0.0,
+      "text": "you cannot buy the best gpus um"
+    },
+    {
+      "start": 5852.28,
+      "duration": 0.0,
+      "text": "you cannot buy the best gpus um there's<01:37:33.280><c> oh</c><01:37:33.440><c> yeah</c><01:37:33.600><c> there's</c><01:37:33.800><c> also</c><01:37:34.040><c> some</c>"
+    },
+    {
+      "start": 5854.229,
+      "duration": 0.0,
+      "text": "there's oh yeah there's also some"
+    },
+    {
+      "start": 5854.239,
+      "duration": 0.0,
+      "text": "there's oh yeah there's also some physical<01:37:34.719><c> limitations</c><01:37:35.719><c> when</c><01:37:35.840><c> you</c><01:37:36.080><c> have</c><01:37:36.280><c> when</c>"
+    },
+    {
+      "start": 5856.35,
+      "duration": 0.0,
+      "text": "physical limitations when you have when"
+    },
+    {
+      "start": 5856.36,
+      "duration": 0.0,
+      "text": "physical limitations when you have when you<01:37:36.440><c> have</c><01:37:36.840><c> multiple</c><01:37:37.199><c> gpus</c><01:37:37.600><c> you</c><01:37:37.719><c> have</c><01:37:37.800><c> to</c>"
+    },
+    {
+      "start": 5857.91,
+      "duration": 0.0,
+      "text": "you have multiple gpus you have to"
+    },
+    {
+      "start": 5857.92,
+      "duration": 0.0,
+      "text": "you have multiple gpus you have to communicate<01:37:38.400><c> between</c><01:37:38.719><c> them</c><01:37:39.119><c> that</c><01:37:39.280><c> takes</c><01:37:39.639><c> time</c>"
+    },
+    {
+      "start": 5860.55,
+      "duration": 0.0,
+      "text": "communicate between them that takes time"
+    },
+    {
+      "start": 5860.56,
+      "duration": 0.0,
+      "text": "communicate between them that takes time um<01:37:40.760><c> so</c><01:37:41.080><c> just</c><01:37:41.239><c> buying</c><01:37:41.520><c> more</c><01:37:41.679><c> gpus</c><01:37:42.159><c> is</c><01:37:42.280><c> not</c><01:37:42.520><c> that</c>"
+    },
+    {
+      "start": 5862.669,
+      "duration": 0.0,
+      "text": "um so just buying more gpus is not that"
+    },
+    {
+      "start": 5862.679,
+      "duration": 0.0,
+      "text": "um so just buying more gpus is not that easy<01:37:43.679><c> um</c><01:37:43.880><c> so</c><01:37:44.080><c> it's</c><01:37:44.280><c> really</c><01:37:44.480><c> important</c><01:37:44.800><c> to</c>"
+    },
+    {
+      "start": 5864.95,
+      "duration": 0.0,
+      "text": "easy um so it's really important to"
+    },
+    {
+      "start": 5864.96,
+      "duration": 0.0,
+      "text": "easy um so it's really important to think<01:37:45.199><c> about</c><01:37:45.560><c> how</c><01:37:45.679><c> do</c><01:37:45.760><c> you</c><01:37:45.880><c> allocate</c>"
+    },
+    {
+      "start": 5866.27,
+      "duration": 0.0,
+      "text": "think about how do you allocate"
+    },
+    {
+      "start": 5866.28,
+      "duration": 0.0,
+      "text": "think about how do you allocate resources<01:37:46.760><c> and</c><01:37:46.880><c> how</c><01:37:47.000><c> do</c><01:37:47.080><c> you</c><01:37:47.199><c> optimize</c><01:37:47.560><c> your</c>"
+    },
+    {
+      "start": 5867.709,
+      "duration": 0.0,
+      "text": "resources and how do you optimize your"
+    },
+    {
+      "start": 5867.719,
+      "duration": 0.0,
+      "text": "resources and how do you optimize your pipeline<01:37:48.159><c> so</c><01:37:48.480><c> system</c><01:37:49.480><c> 101</c><01:37:50.280><c> on</c><01:37:50.880><c> gpus</c><01:37:51.440><c> I'm</c><01:37:51.560><c> sorry</c>"
+    },
+    {
+      "start": 5871.79,
+      "duration": 0.0,
+      "text": "pipeline so system 101 on gpus I'm sorry"
+    },
+    {
+      "start": 5871.8,
+      "duration": 0.0,
+      "text": "pipeline so system 101 on gpus I'm sorry I'm<01:37:51.880><c> going</c><01:37:52.239><c> slightly</c><01:37:52.679><c> faster</c><01:37:53.000><c> I</c><01:37:53.119><c> hope</c><01:37:53.280><c> for</c>"
+    },
+    {
+      "start": 5873.55,
+      "duration": 0.0,
+      "text": "I'm going slightly faster I hope for"
+    },
+    {
+      "start": 5873.56,
+      "duration": 0.0,
+      "text": "I'm going slightly faster I hope for that<01:37:53.679><c> some</c><01:37:53.840><c> of</c><01:37:53.960><c> you</c><01:37:54.199><c> at</c><01:37:54.320><c> least</c><01:37:54.520><c> can</c><01:37:54.719><c> follow</c><01:37:55.719><c> uh</c>"
+    },
+    {
+      "start": 5875.83,
+      "duration": 0.0,
+      "text": "that some of you at least can follow uh"
+    },
+    {
+      "start": 5875.84,
+      "duration": 0.0,
+      "text": "that some of you at least can follow uh gpus<01:37:56.280><c> are</c><01:37:56.440><c> basically</c><01:37:56.760><c> optimized</c><01:37:57.199><c> for</c>"
+    },
+    {
+      "start": 5877.35,
+      "duration": 0.0,
+      "text": "gpus are basically optimized for"
+    },
+    {
+      "start": 5877.36,
+      "duration": 0.0,
+      "text": "gpus are basically optimized for throughput<01:37:58.199><c> CPUs</c><01:37:58.840><c> are</c><01:37:59.280><c> optimized</c><01:38:00.280><c> uh</c><01:38:00.360><c> for</c>"
+    },
+    {
+      "start": 5880.589,
+      "duration": 0.0,
+      "text": "throughput CPUs are optimized uh for"
+    },
+    {
+      "start": 5880.599,
+      "duration": 0.0,
+      "text": "throughput CPUs are optimized uh for latency<01:38:01.599><c> so</c><01:38:02.000><c> gpus</c><01:38:02.480><c> the</c><01:38:02.599><c> way</c><01:38:02.719><c> you</c><01:38:02.840><c> have</c><01:38:02.920><c> to</c>"
+    },
+    {
+      "start": 5883.07,
+      "duration": 0.0,
+      "text": "latency so gpus the way you have to"
+    },
+    {
+      "start": 5883.08,
+      "duration": 0.0,
+      "text": "latency so gpus the way you have to think<01:38:03.239><c> about</c><01:38:03.440><c> it</c><01:38:03.599><c> is</c><01:38:03.719><c> that</c><01:38:03.920><c> there's</c><01:38:04.239><c> one</c><01:38:04.520><c> Comm</c>"
+    },
+    {
+      "start": 5884.83,
+      "duration": 0.0,
+      "text": "think about it is that there's one Comm"
+    },
+    {
+      "start": 5884.84,
+      "duration": 0.0,
+      "text": "think about it is that there's one Comm there's<01:38:05.239><c> one</c><01:38:05.480><c> command</c><01:38:05.840><c> that</c><01:38:05.960><c> is</c><01:38:06.119><c> run</c><01:38:06.679><c> on</c><01:38:07.000><c> many</c>"
+    },
+    {
+      "start": 5887.27,
+      "duration": 0.0,
+      "text": "there's one command that is run on many"
+    },
+    {
+      "start": 5887.28,
+      "duration": 0.0,
+      "text": "there's one command that is run on many many<01:38:07.480><c> Calles</c><01:38:07.800><c> at</c><01:38:07.920><c> the</c><01:38:08.040><c> same</c><01:38:08.239><c> time</c><01:38:08.719><c> on</c>"
+    },
+    {
+      "start": 5888.95,
+      "duration": 0.0,
+      "text": "many Calles at the same time on"
+    },
+    {
+      "start": 5888.96,
+      "duration": 0.0,
+      "text": "many Calles at the same time on different<01:38:09.239><c> type</c><01:38:09.480><c> of</c><01:38:09.679><c> data</c><01:38:10.639><c> um</c><01:38:11.520><c> so</c><01:38:12.000><c> this</c><01:38:12.080><c> is</c><01:38:12.320><c> how</c>"
+    },
+    {
+      "start": 5892.43,
+      "duration": 0.0,
+      "text": "different type of data um so this is how"
+    },
+    {
+      "start": 5892.44,
+      "duration": 0.0,
+      "text": "different type of data um so this is how you<01:38:12.520><c> see</c><01:38:12.679><c> a</c><01:38:12.800><c> GPU</c><01:38:13.239><c> you</c><01:38:13.320><c> see</c><01:38:13.520><c> there</c><01:38:13.639><c> are</c><01:38:13.920><c> many</c>"
+    },
+    {
+      "start": 5894.149,
+      "duration": 0.0,
+      "text": "you see a GPU you see there are many"
+    },
+    {
+      "start": 5894.159,
+      "duration": 0.0,
+      "text": "you see a GPU you see there are many different<01:38:14.400><c> CES</c><01:38:14.760><c> we</c><01:38:14.920><c> call</c><01:38:15.119><c> them</c><01:38:15.599><c> streaming</c>"
+    },
+    {
+      "start": 5896.47,
+      "duration": 0.0,
+      "text": "different CES we call them streaming"
+    },
+    {
+      "start": 5896.48,
+      "duration": 0.0,
+      "text": "different CES we call them streaming multiprocessors<01:38:17.480><c> which</c><01:38:17.599><c> is</c><01:38:17.760><c> very</c><01:38:17.920><c> different</c>"
+    },
+    {
+      "start": 5898.149,
+      "duration": 0.0,
+      "text": "multiprocessors which is very different"
+    },
+    {
+      "start": 5898.159,
+      "duration": 0.0,
+      "text": "multiprocessors which is very different than<01:38:18.280><c> the</c><01:38:18.440><c> usual</c><01:38:19.239><c> CPU</c><01:38:19.679><c> architecture</c><01:38:20.239><c> so</c><01:38:20.440><c> just</c>"
+    },
+    {
+      "start": 5900.589,
+      "duration": 0.0,
+      "text": "than the usual CPU architecture so just"
+    },
+    {
+      "start": 5900.599,
+      "duration": 0.0,
+      "text": "than the usual CPU architecture so just think<01:38:21.080><c> High</c><01:38:21.719><c> throughput</c><01:38:22.719><c> paralyzation</c><01:38:23.480><c> for</c>"
+    },
+    {
+      "start": 5903.83,
+      "duration": 0.0,
+      "text": "think High throughput paralyzation for"
+    },
+    {
+      "start": 5903.84,
+      "duration": 0.0,
+      "text": "think High throughput paralyzation for gpus<01:38:24.840><c> uh</c><01:38:24.960><c> gpus</c><01:38:25.360><c> are</c><01:38:25.520><c> optimized</c><01:38:25.920><c> for</c><01:38:26.119><c> fast</c>"
+    },
+    {
+      "start": 5906.35,
+      "duration": 0.0,
+      "text": "gpus uh gpus are optimized for fast"
+    },
+    {
+      "start": 5906.36,
+      "duration": 0.0,
+      "text": "gpus uh gpus are optimized for fast matrix<01:38:26.840><c> multiplication</c><01:38:27.840><c> so</c><01:38:28.400><c> every</c><01:38:28.679><c> time</c><01:38:28.840><c> you</c>"
+    },
+    {
+      "start": 5908.91,
+      "duration": 0.0,
+      "text": "matrix multiplication so every time you"
+    },
+    {
+      "start": 5908.92,
+      "duration": 0.0,
+      "text": "matrix multiplication so every time you will<01:38:29.119><c> do</c><01:38:29.560><c> uh</c><01:38:29.639><c> you</c><01:38:29.719><c> will</c><01:38:29.840><c> do</c><01:38:30.000><c> something</c><01:38:30.199><c> on</c><01:38:30.360><c> GPU</c>"
+    },
+    {
+      "start": 5910.75,
+      "duration": 0.0,
+      "text": "will do uh you will do something on GPU"
+    },
+    {
+      "start": 5910.76,
+      "duration": 0.0,
+      "text": "will do uh you will do something on GPU if<01:38:30.840><c> you</c><01:38:30.920><c> can</c><01:38:31.040><c> do</c><01:38:31.159><c> it</c><01:38:31.280><c> with</c><01:38:31.400><c> a</c><01:38:32.199><c> a</c><01:38:32.360><c> matrix</c>"
+    },
+    {
+      "start": 5912.79,
+      "duration": 0.0,
+      "text": "if you can do it with a a matrix"
+    },
+    {
+      "start": 5912.8,
+      "duration": 0.0,
+      "text": "if you can do it with a a matrix multiplication<01:38:33.440><c> it's</c><01:38:33.599><c> going</c><01:38:33.679><c> to</c><01:38:33.760><c> be</c><01:38:33.920><c> 10</c><01:38:34.159><c> times</c>"
+    },
+    {
+      "start": 5914.47,
+      "duration": 0.0,
+      "text": "multiplication it's going to be 10 times"
+    },
+    {
+      "start": 5914.48,
+      "duration": 0.0,
+      "text": "multiplication it's going to be 10 times faster<01:38:35.119><c> than</c><01:38:35.320><c> with</c><01:38:35.520><c> anything</c><01:38:35.800><c> else</c><01:38:36.719><c> uh</c><01:38:36.880><c> that</c>"
+    },
+    {
+      "start": 5916.99,
+      "duration": 0.0,
+      "text": "faster than with anything else uh that"
+    },
+    {
+      "start": 5917.0,
+      "duration": 0.0,
+      "text": "faster than with anything else uh that is<01:38:37.119><c> a</c><01:38:37.199><c> little</c><01:38:37.360><c> bit</c><01:38:37.480><c> annoying</c><01:38:37.920><c> because</c><01:38:38.040><c> it</c>"
+    },
+    {
+      "start": 5918.109,
+      "duration": 0.0,
+      "text": "is a little bit annoying because it"
+    },
+    {
+      "start": 5918.119,
+      "duration": 0.0,
+      "text": "is a little bit annoying because it means<01:38:38.360><c> that</c><01:38:38.520><c> we're</c><01:38:38.760><c> kind</c><01:38:38.920><c> of</c><01:38:39.800><c> uh</c><01:38:40.080><c> bottlenecked</c>"
+    },
+    {
+      "start": 5920.75,
+      "duration": 0.0,
+      "text": "means that we're kind of uh bottlenecked"
+    },
+    {
+      "start": 5920.76,
+      "duration": 0.0,
+      "text": "means that we're kind of uh bottlenecked to<01:38:40.920><c> doing</c><01:38:41.239><c> anything</c><01:38:41.599><c> with</c><01:38:42.119><c> Matrix</c>"
+    },
+    {
+      "start": 5923.109,
+      "duration": 0.0,
+      "text": "to doing anything with Matrix"
+    },
+    {
+      "start": 5923.119,
+      "duration": 0.0,
+      "text": "to doing anything with Matrix multiplications<01:38:44.119><c> um</c><01:38:44.360><c> another</c><01:38:44.639><c> thing</c><01:38:44.760><c> to</c><01:38:44.880><c> note</c>"
+    },
+    {
+      "start": 5925.109,
+      "duration": 0.0,
+      "text": "multiplications um another thing to note"
+    },
+    {
+      "start": 5925.119,
+      "duration": 0.0,
+      "text": "multiplications um another thing to note with<01:38:45.280><c> gpus</c><01:38:46.199><c> is</c><01:38:46.360><c> that</c><01:38:46.639><c> compute</c><01:38:47.480><c> has</c><01:38:47.599><c> been</c>"
+    },
+    {
+      "start": 5927.75,
+      "duration": 0.0,
+      "text": "with gpus is that compute has been"
+    },
+    {
+      "start": 5927.76,
+      "duration": 0.0,
+      "text": "with gpus is that compute has been improving<01:38:48.199><c> faster</c><01:38:48.560><c> than</c><01:38:48.760><c> memory</c><01:38:49.159><c> and</c>"
+    },
+    {
+      "start": 5929.47,
+      "duration": 0.0,
+      "text": "improving faster than memory and"
+    },
+    {
+      "start": 5929.48,
+      "duration": 0.0,
+      "text": "improving faster than memory and communication<01:38:50.480><c> so</c><01:38:50.880><c> right</c><01:38:51.080><c> now</c><01:38:51.840><c> gpus</c><01:38:52.840><c> usually</c>"
+    },
+    {
+      "start": 5933.589,
+      "duration": 0.0,
+      "text": "communication so right now gpus usually"
+    },
+    {
+      "start": 5933.599,
+      "duration": 0.0,
+      "text": "communication so right now gpus usually are<01:38:53.880><c> hard</c><01:38:54.159><c> to</c><01:38:54.679><c> keep</c><01:38:55.560><c> uh</c><01:38:56.000><c> like</c><01:38:56.119><c> the</c><01:38:56.280><c> data</c><01:38:56.520><c> that</c>"
+    },
+    {
+      "start": 5936.629,
+      "duration": 0.0,
+      "text": "are hard to keep uh like the data that"
+    },
+    {
+      "start": 5936.639,
+      "duration": 0.0,
+      "text": "are hard to keep uh like the data that you<01:38:56.719><c> send</c><01:38:57.000><c> that</c><01:38:57.400><c> send</c><01:38:57.639><c> to</c><01:38:57.800><c> gpus</c><01:38:58.719><c> is</c><01:38:58.920><c> actually</c>"
+    },
+    {
+      "start": 5939.189,
+      "duration": 0.0,
+      "text": "you send that send to gpus is actually"
+    },
+    {
+      "start": 5939.199,
+      "duration": 0.0,
+      "text": "you send that send to gpus is actually hard<01:38:59.400><c> to</c><01:38:59.560><c> keep</c><01:38:59.719><c> up</c><01:38:59.880><c> with</c><01:39:00.000><c> the</c><01:39:00.119><c> processess</c><01:39:00.760><c> so</c>"
+    },
+    {
+      "start": 5940.95,
+      "duration": 0.0,
+      "text": "hard to keep up with the processess so"
+    },
+    {
+      "start": 5940.96,
+      "duration": 0.0,
+      "text": "hard to keep up with the processess so most<01:39:01.159><c> of</c><01:39:01.280><c> your</c><01:39:01.440><c> gpus</c><01:39:01.840><c> are</c><01:39:02.000><c> actually</c><01:39:02.159><c> going</c><01:39:02.280><c> to</c>"
+    },
+    {
+      "start": 5942.39,
+      "duration": 0.0,
+      "text": "most of your gpus are actually going to"
+    },
+    {
+      "start": 5942.4,
+      "duration": 0.0,
+      "text": "most of your gpus are actually going to be<01:39:02.560><c> idle</c><01:39:03.040><c> if</c><01:39:03.159><c> you</c><01:39:03.280><c> just</c><01:39:03.440><c> run</c><01:39:03.719><c> normal</c><01:39:04.080><c> code</c><01:39:04.920><c> if</c>"
+    },
+    {
+      "start": 5944.99,
+      "duration": 0.0,
+      "text": "be idle if you just run normal code if"
+    },
+    {
+      "start": 5945.0,
+      "duration": 0.0,
+      "text": "be idle if you just run normal code if you<01:39:05.080><c> don't</c><01:39:05.280><c> optimize</c><01:39:05.679><c> your</c><01:39:05.840><c> code</c><01:39:06.320><c> so</c>"
+    },
+    {
+      "start": 5946.589,
+      "duration": 0.0,
+      "text": "you don't optimize your code so"
+    },
+    {
+      "start": 5946.599,
+      "duration": 0.0,
+      "text": "you don't optimize your code so communication<01:39:07.560><c> and</c><01:39:07.719><c> this</c><01:39:07.920><c> will</c><01:39:08.440><c> continue</c>"
+    },
+    {
+      "start": 5949.229,
+      "duration": 0.0,
+      "text": "communication and this will continue"
+    },
+    {
+      "start": 5949.239,
+      "duration": 0.0,
+      "text": "communication and this will continue over<01:39:10.119><c> time</c><01:39:11.119><c> another</c><01:39:11.400><c> thing</c><01:39:11.480><c> to</c><01:39:11.599><c> know</c><01:39:11.800><c> about</c>"
+    },
+    {
+      "start": 5951.95,
+      "duration": 0.0,
+      "text": "over time another thing to know about"
+    },
+    {
+      "start": 5951.96,
+      "duration": 0.0,
+      "text": "over time another thing to know about gpus<01:39:12.520><c> is</c><01:39:12.599><c> that</c><01:39:12.719><c> there's</c><01:39:12.840><c> a</c><01:39:13.000><c> memory</c><01:39:13.280><c> hierarchy</c>"
+    },
+    {
+      "start": 5953.79,
+      "duration": 0.0,
+      "text": "gpus is that there's a memory hierarchy"
+    },
+    {
+      "start": 5953.8,
+      "duration": 0.0,
+      "text": "gpus is that there's a memory hierarchy this<01:39:13.880><c> is</c><01:39:14.000><c> the</c><01:39:14.119><c> same</c><01:39:14.280><c> thing</c><01:39:14.440><c> actually</c><01:39:14.679><c> with</c>"
+    },
+    {
+      "start": 5954.79,
+      "duration": 0.0,
+      "text": "this is the same thing actually with"
+    },
+    {
+      "start": 5954.8,
+      "duration": 0.0,
+      "text": "this is the same thing actually with CPUs<01:39:15.520><c> but</c><01:39:15.679><c> basically</c><01:39:16.040><c> the</c><01:39:16.159><c> closer</c><01:39:16.800><c> you</c><01:39:16.920><c> are</c><01:39:17.080><c> to</c>"
+    },
+    {
+      "start": 5957.189,
+      "duration": 0.0,
+      "text": "CPUs but basically the closer you are to"
+    },
+    {
+      "start": 5957.199,
+      "duration": 0.0,
+      "text": "CPUs but basically the closer you are to your<01:39:17.400><c> cuse</c><01:39:17.760><c> the</c><01:39:17.880><c> less</c><01:39:18.040><c> memory</c><01:39:18.440><c> there</c><01:39:18.639><c> is</c><01:39:19.119><c> but</c>"
+    },
+    {
+      "start": 5959.27,
+      "duration": 0.0,
+      "text": "your cuse the less memory there is but"
+    },
+    {
+      "start": 5959.28,
+      "duration": 0.0,
+      "text": "your cuse the less memory there is but the<01:39:19.480><c> faster</c><01:39:19.840><c> things</c><01:39:20.080><c> run</c><01:39:20.679><c> if</c><01:39:20.800><c> you're</c><01:39:21.040><c> further</c>"
+    },
+    {
+      "start": 5961.709,
+      "duration": 0.0,
+      "text": "the faster things run if you're further"
+    },
+    {
+      "start": 5961.719,
+      "duration": 0.0,
+      "text": "the faster things run if you're further more<01:39:21.920><c> memory</c><01:39:22.360><c> slower</c>"
+    },
+    {
+      "start": 5963.99,
+      "duration": 0.0,
+      "text": "more memory slower"
+    },
+    {
+      "start": 5964.0,
+      "duration": 0.0,
+      "text": "more memory slower um<01:39:25.000><c> okay</c><01:39:25.119><c> I'm</c><01:39:25.239><c> going</c><01:39:25.320><c> to</c><01:39:25.400><c> skip</c><01:39:25.679><c> that</c><01:39:26.320><c> okay</c>"
+    },
+    {
+      "start": 5966.51,
+      "duration": 0.0,
+      "text": "um okay I'm going to skip that okay"
+    },
+    {
+      "start": 5966.52,
+      "duration": 0.0,
+      "text": "um okay I'm going to skip that okay actually<01:39:26.719><c> I'm</c><01:39:26.840><c> going</c><01:39:26.920><c> to</c><01:39:27.040><c> say</c><01:39:27.159><c> it</c><01:39:27.960><c> I</c><01:39:28.080><c> told</c><01:39:28.280><c> you</c>"
+    },
+    {
+      "start": 5968.43,
+      "duration": 0.0,
+      "text": "actually I'm going to say it I told you"
+    },
+    {
+      "start": 5968.44,
+      "duration": 0.0,
+      "text": "actually I'm going to say it I told you about<01:39:28.760><c> this</c><01:39:29.080><c> uh</c><01:39:29.239><c> the</c><01:39:29.440><c> fact</c><01:39:29.599><c> of</c><01:39:30.040><c> communication</c>"
+    },
+    {
+      "start": 5971.03,
+      "duration": 0.0,
+      "text": "about this uh the fact of communication"
+    },
+    {
+      "start": 5971.04,
+      "duration": 0.0,
+      "text": "about this uh the fact of communication uh<01:39:31.159><c> the</c><01:39:31.360><c> metric</c><01:39:31.639><c> that</c><01:39:31.760><c> people</c><01:39:31.960><c> usually</c><01:39:32.239><c> look</c>"
+    },
+    {
+      "start": 5972.43,
+      "duration": 0.0,
+      "text": "uh the metric that people usually look"
+    },
+    {
+      "start": 5972.44,
+      "duration": 0.0,
+      "text": "uh the metric that people usually look at<01:39:32.719><c> is</c><01:39:32.880><c> model</c><01:39:33.199><c> flop</c><01:39:33.560><c> utilization</c><01:39:34.440><c> so</c><01:39:34.599><c> what</c><01:39:34.719><c> is</c>"
+    },
+    {
+      "start": 5974.87,
+      "duration": 0.0,
+      "text": "at is model flop utilization so what is"
+    },
+    {
+      "start": 5974.88,
+      "duration": 0.0,
+      "text": "at is model flop utilization so what is the<01:39:35.040><c> theoretical</c><01:39:35.520><c> maximum</c><01:39:36.000><c> that</c><01:39:36.440><c> GPU</c><01:39:36.840><c> could</c>"
+    },
+    {
+      "start": 5976.99,
+      "duration": 0.0,
+      "text": "the theoretical maximum that GPU could"
+    },
+    {
+      "start": 5977.0,
+      "duration": 0.0,
+      "text": "the theoretical maximum that GPU could run<01:39:37.320><c> at</c><01:39:37.560><c> no</c><01:39:37.760><c> more</c><01:39:37.960><c> flops</c><01:39:38.280><c> that</c><01:39:38.360><c> you</c><01:39:38.440><c> could</c><01:39:38.560><c> use</c>"
+    },
+    {
+      "start": 5978.79,
+      "duration": 0.0,
+      "text": "run at no more flops that you could use"
+    },
+    {
+      "start": 5978.8,
+      "duration": 0.0,
+      "text": "run at no more flops that you could use per<01:39:38.960><c> second</c><01:39:39.880><c> divide</c><01:39:40.320><c> sorry</c><01:39:40.639><c> the</c><01:39:40.800><c> number</c><01:39:41.000><c> of</c><01:39:41.239><c> OB</c>"
+    },
+    {
+      "start": 5981.629,
+      "duration": 0.0,
+      "text": "per second divide sorry the number of OB"
+    },
+    {
+      "start": 5981.639,
+      "duration": 0.0,
+      "text": "per second divide sorry the number of OB observed<01:39:42.199><c> through</c><01:39:42.560><c> put</c><01:39:42.679><c> divided</c><01:39:43.000><c> by</c><01:39:43.119><c> this</c>"
+    },
+    {
+      "start": 5983.51,
+      "duration": 0.0,
+      "text": "observed through put divided by this"
+    },
+    {
+      "start": 5983.52,
+      "duration": 0.0,
+      "text": "observed through put divided by this theoretical<01:39:44.520><c> um</c><01:39:45.159><c> maximum</c><01:39:46.159><c> and</c><01:39:46.480><c> in</c><01:39:46.639><c> general</c><01:39:47.000><c> if</c>"
+    },
+    {
+      "start": 5987.109,
+      "duration": 0.0,
+      "text": "theoretical um maximum and in general if"
+    },
+    {
+      "start": 5987.119,
+      "duration": 0.0,
+      "text": "theoretical um maximum and in general if you<01:39:47.440><c> reach</c><01:39:47.760><c> 50%</c><01:39:48.320><c> you're</c><01:39:48.520><c> very</c><01:39:48.719><c> happy</c><01:39:49.440><c> like</c>"
+    },
+    {
+      "start": 5989.629,
+      "duration": 0.0,
+      "text": "you reach 50% you're very happy like"
+    },
+    {
+      "start": 5989.639,
+      "duration": 0.0,
+      "text": "you reach 50% you're very happy like Facebook<01:39:50.000><c> I</c><01:39:50.119><c> looked</c><01:39:50.320><c> at</c><01:39:50.440><c> Lama</c><01:39:50.840><c> was</c><01:39:50.960><c> at</c><01:39:51.119><c> 45</c><01:39:51.679><c> or</c>"
+    },
+    {
+      "start": 5991.83,
+      "duration": 0.0,
+      "text": "Facebook I looked at Lama was at 45 or"
+    },
+    {
+      "start": 5991.84,
+      "duration": 0.0,
+      "text": "Facebook I looked at Lama was at 45 or something<01:39:52.119><c> like</c><01:39:52.320><c> this</c><01:39:52.800><c> so</c><01:39:53.199><c> that</c><01:39:53.400><c> that</c><01:39:53.560><c> means</c>"
+    },
+    {
+      "start": 5994.149,
+      "duration": 0.0,
+      "text": "something like this so that that means"
+    },
+    {
+      "start": 5994.159,
+      "duration": 0.0,
+      "text": "something like this so that that means that<01:39:54.440><c> data</c><01:39:54.719><c> doesn't</c><01:39:55.000><c> come</c><01:39:55.280><c> fast</c><01:39:55.560><c> enough</c><01:39:56.000><c> even</c>"
+    },
+    {
+      "start": 5996.229,
+      "duration": 0.0,
+      "text": "that data doesn't come fast enough even"
+    },
+    {
+      "start": 5996.239,
+      "duration": 0.0,
+      "text": "that data doesn't come fast enough even for<01:39:56.480><c> these</c><01:39:56.679><c> big</c>"
+    },
+    {
+      "start": 5997.99,
+      "duration": 0.0,
+      "text": "for these big"
+    },
+    {
+      "start": 5998.0,
+      "duration": 0.0,
+      "text": "for these big companies<01:39:59.000><c> so</c><01:39:59.440><c> one</c><01:39:59.760><c> simple</c><01:40:00.080><c> trick</c><01:40:00.440><c> and</c><01:40:00.599><c> that</c>"
+    },
+    {
+      "start": 6000.709,
+      "duration": 0.0,
+      "text": "companies so one simple trick and that"
+    },
+    {
+      "start": 6000.719,
+      "duration": 0.0,
+      "text": "companies so one simple trick and that might<01:40:00.880><c> be</c><01:40:01.040><c> the</c><01:40:01.119><c> only</c><01:40:01.360><c> one</c><01:40:01.480><c> I'm</c><01:40:01.599><c> going</c><01:40:01.679><c> to</c><01:40:02.040><c> tell</c>"
+    },
+    {
+      "start": 6002.149,
+      "duration": 0.0,
+      "text": "might be the only one I'm going to tell"
+    },
+    {
+      "start": 6002.159,
+      "duration": 0.0,
+      "text": "might be the only one I'm going to tell you<01:40:02.320><c> about</c><01:40:02.800><c> is</c><01:40:02.960><c> low</c><01:40:03.480><c> Precision</c><01:40:04.480><c> one</c><01:40:04.760><c> simple</c>"
+    },
+    {
+      "start": 6005.149,
+      "duration": 0.0,
+      "text": "you about is low Precision one simple"
+    },
+    {
+      "start": 6005.159,
+      "duration": 0.0,
+      "text": "you about is low Precision one simple idea<01:40:05.840><c> is</c><01:40:06.040><c> that</c><01:40:06.320><c> well</c><01:40:06.599><c> if</c><01:40:06.760><c> I'm</c><01:40:06.880><c> going</c><01:40:07.000><c> to</c><01:40:07.159><c> put</c><01:40:07.400><c> my</c>"
+    },
+    {
+      "start": 6007.51,
+      "duration": 0.0,
+      "text": "idea is that well if I'm going to put my"
+    },
+    {
+      "start": 6007.52,
+      "duration": 0.0,
+      "text": "idea is that well if I'm going to put my floats<01:40:08.080><c> in</c><01:40:08.239><c> lower</c><01:40:08.599><c> Precision</c><01:40:09.480><c> then</c><01:40:09.639><c> there's</c>"
+    },
+    {
+      "start": 6009.83,
+      "duration": 0.0,
+      "text": "floats in lower Precision then there's"
+    },
+    {
+      "start": 6009.84,
+      "duration": 0.0,
+      "text": "floats in lower Precision then there's going<01:40:09.920><c> to</c><01:40:10.000><c> be</c><01:40:10.119><c> fewer</c><01:40:10.480><c> bits</c><01:40:10.800><c> that</c><01:40:10.880><c> I</c><01:40:11.000><c> have</c><01:40:11.119><c> to</c>"
+    },
+    {
+      "start": 6011.229,
+      "duration": 0.0,
+      "text": "going to be fewer bits that I have to"
+    },
+    {
+      "start": 6011.239,
+      "duration": 0.0,
+      "text": "going to be fewer bits that I have to send<01:40:11.480><c> to</c><01:40:11.639><c> my</c><01:40:11.760><c> gpus</c><01:40:12.480><c> if</c><01:40:12.639><c> there's</c><01:40:12.840><c> fewer</c><01:40:13.119><c> bits</c>"
+    },
+    {
+      "start": 6013.39,
+      "duration": 0.0,
+      "text": "send to my gpus if there's fewer bits"
+    },
+    {
+      "start": 6013.4,
+      "duration": 0.0,
+      "text": "send to my gpus if there's fewer bits it's<01:40:13.639><c> faster</c><01:40:13.920><c> communication</c><01:40:14.840><c> lower</c><01:40:15.119><c> memory</c>"
+    },
+    {
+      "start": 6015.43,
+      "duration": 0.0,
+      "text": "it's faster communication lower memory"
+    },
+    {
+      "start": 6015.44,
+      "duration": 0.0,
+      "text": "it's faster communication lower memory consumption<01:40:15.960><c> things</c><01:40:16.119><c> are</c><01:40:16.239><c> going</c><01:40:16.360><c> to</c><01:40:16.480><c> go</c>"
+    },
+    {
+      "start": 6016.79,
+      "duration": 0.0,
+      "text": "consumption things are going to go"
+    },
+    {
+      "start": 6016.8,
+      "duration": 0.0,
+      "text": "consumption things are going to go faster<01:40:17.800><c> uh</c><01:40:18.000><c> and</c><01:40:18.080><c> for</c><01:40:18.280><c> deep</c><01:40:18.520><c> learning</c><01:40:18.920><c> it</c><01:40:19.040><c> just</c>"
+    },
+    {
+      "start": 6019.229,
+      "duration": 0.0,
+      "text": "faster uh and for deep learning it just"
+    },
+    {
+      "start": 6019.239,
+      "duration": 0.0,
+      "text": "faster uh and for deep learning it just happens<01:40:19.560><c> that</c><01:40:20.000><c> de</c><01:40:20.480><c> decimal</c><01:40:21.280><c> is</c><01:40:21.440><c> not</c><01:40:21.679><c> that</c>"
+    },
+    {
+      "start": 6021.87,
+      "duration": 0.0,
+      "text": "happens that de decimal is not that"
+    },
+    {
+      "start": 6021.88,
+      "duration": 0.0,
+      "text": "happens that de decimal is not that important<01:40:22.880><c> uh</c><01:40:23.040><c> so</c><01:40:23.360><c> so</c><01:40:23.840><c> when</c><01:40:24.000><c> you</c><01:40:24.159><c> do</c><01:40:24.360><c> matrix</c>"
+    },
+    {
+      "start": 6024.709,
+      "duration": 0.0,
+      "text": "important uh so so when you do matrix"
+    },
+    {
+      "start": 6024.719,
+      "duration": 0.0,
+      "text": "important uh so so when you do matrix multiplication<01:40:25.599><c> when</c><01:40:25.719><c> you</c><01:40:25.840><c> do</c><01:40:26.040><c> like</c><01:40:26.159><c> for</c>"
+    },
+    {
+      "start": 6026.27,
+      "duration": 0.0,
+      "text": "multiplication when you do like for"
+    },
+    {
+      "start": 6026.28,
+      "duration": 0.0,
+      "text": "multiplication when you do like for example<01:40:26.560><c> SGD</c><01:40:27.000><c> there's</c><01:40:27.199><c> already</c><01:40:27.560><c> so</c><01:40:27.719><c> much</c>"
+    },
+    {
+      "start": 6027.95,
+      "duration": 0.0,
+      "text": "example SGD there's already so much"
+    },
+    {
+      "start": 6027.96,
+      "duration": 0.0,
+      "text": "example SGD there's already so much noise<01:40:28.560><c> that</c><01:40:28.719><c> if</c><01:40:28.800><c> you</c><01:40:28.960><c> update</c><01:40:29.320><c> something</c><01:40:29.639><c> by</c>"
+    },
+    {
+      "start": 6029.75,
+      "duration": 0.0,
+      "text": "noise that if you update something by"
+    },
+    {
+      "start": 6029.76,
+      "duration": 0.0,
+      "text": "noise that if you update something by 0.01<01:40:30.760><c> or</c>"
+    },
+    {
+      "start": 6031.87,
+      "duration": 0.0,
+      "text": "0.01 or"
+    },
+    {
+      "start": 6031.88,
+      "duration": 0.0,
+      "text": "0.01 or 0.015<01:40:32.880><c> who</c><01:40:33.040><c> cares</c><01:40:33.840><c> uh</c><01:40:33.920><c> so</c><01:40:34.119><c> basically</c><01:40:34.520><c> instead</c>"
+    },
+    {
+      "start": 6034.79,
+      "duration": 0.0,
+      "text": "0.015 who cares uh so basically instead"
+    },
+    {
+      "start": 6034.8,
+      "duration": 0.0,
+      "text": "0.015 who cares uh so basically instead of<01:40:34.960><c> using</c><01:40:35.960><c> uh</c><01:40:36.119><c> 32</c><01:40:36.639><c> bits</c><01:40:37.199><c> per</c><01:40:37.360><c> float</c><01:40:37.840><c> which</c><01:40:38.000><c> is</c>"
+    },
+    {
+      "start": 6038.51,
+      "duration": 0.0,
+      "text": "of using uh 32 bits per float which is"
+    },
+    {
+      "start": 6038.52,
+      "duration": 0.0,
+      "text": "of using uh 32 bits per float which is um<01:40:38.920><c> what</c><01:40:39.080><c> people</c><01:40:39.520><c> used</c><01:40:39.760><c> to</c><01:40:39.920><c> use</c><01:40:40.159><c> or</c><01:40:40.360><c> 64</c><01:40:40.920><c> for</c>"
+    },
+    {
+      "start": 6041.07,
+      "duration": 0.0,
+      "text": "um what people used to use or 64 for"
+    },
+    {
+      "start": 6041.08,
+      "duration": 0.0,
+      "text": "um what people used to use or 64 for example<01:40:41.400><c> which</c><01:40:41.480><c> is</c><01:40:41.639><c> what</c><01:40:41.840><c> you</c><01:40:42.000><c> would</c><01:40:42.199><c> use</c><01:40:42.639><c> in</c>"
+    },
+    {
+      "start": 6042.83,
+      "duration": 0.0,
+      "text": "example which is what you would use in"
+    },
+    {
+      "start": 6042.84,
+      "duration": 0.0,
+      "text": "example which is what you would use in other<01:40:43.119><c> domains</c><01:40:43.639><c> you</c><01:40:43.760><c> use</c><01:40:44.000><c> 16</c><01:40:44.440><c> bits</c><01:40:45.119><c> uh</c><01:40:45.239><c> for</c>"
+    },
+    {
+      "start": 6045.39,
+      "duration": 0.0,
+      "text": "other domains you use 16 bits uh for"
+    },
+    {
+      "start": 6045.4,
+      "duration": 0.0,
+      "text": "other domains you use 16 bits uh for matrix<01:40:45.760><c> multiplication</c><01:40:46.320><c> so</c><01:40:46.480><c> for</c><01:40:46.599><c> every</c><01:40:46.760><c> float</c>"
+    },
+    {
+      "start": 6047.03,
+      "duration": 0.0,
+      "text": "matrix multiplication so for every float"
+    },
+    {
+      "start": 6047.04,
+      "duration": 0.0,
+      "text": "matrix multiplication so for every float you<01:40:47.119><c> use</c><01:40:47.280><c> 16</c><01:40:48.000><c> bits</c><01:40:49.000><c> um</c><01:40:49.840><c> and</c><01:40:49.960><c> for</c><01:40:50.159><c> training</c><01:40:50.560><c> you</c>"
+    },
+    {
+      "start": 6050.709,
+      "duration": 0.0,
+      "text": "you use 16 bits um and for training you"
+    },
+    {
+      "start": 6050.719,
+      "duration": 0.0,
+      "text": "you use 16 bits um and for training you have<01:40:50.920><c> this</c><01:40:51.080><c> type</c><01:40:51.280><c> of</c><01:40:51.520><c> like</c><01:40:52.440><c> uh</c><01:40:52.599><c> what</c><01:40:52.719><c> we</c><01:40:52.840><c> call</c>"
+    },
+    {
+      "start": 6052.99,
+      "duration": 0.0,
+      "text": "have this type of like uh what we call"
+    },
+    {
+      "start": 6053.0,
+      "duration": 0.0,
+      "text": "have this type of like uh what we call aut<01:40:53.280><c> atic</c><01:40:53.520><c> mix</c><01:40:53.760><c> Precision</c><01:40:54.239><c> which</c><01:40:54.360><c> is</c><01:40:54.520><c> that</c><01:40:55.199><c> uh</c>"
+    },
+    {
+      "start": 6055.35,
+      "duration": 0.0,
+      "text": "aut atic mix Precision which is that uh"
+    },
+    {
+      "start": 6055.36,
+      "duration": 0.0,
+      "text": "aut atic mix Precision which is that uh some<01:40:55.599><c> of</c><01:40:55.719><c> the</c><01:40:55.840><c> things</c><01:40:56.080><c> are</c><01:40:56.199><c> in</c><01:40:56.360><c> 32</c><01:40:56.760><c> bits</c><01:40:57.199><c> others</c>"
+    },
+    {
+      "start": 6057.47,
+      "duration": 0.0,
+      "text": "some of the things are in 32 bits others"
+    },
+    {
+      "start": 6057.48,
+      "duration": 0.0,
+      "text": "some of the things are in 32 bits others are<01:40:57.599><c> in</c><01:40:57.719><c> 60</c><01:40:58.040><c> bit</c><01:40:58.679><c> in</c><01:40:58.840><c> 16</c><01:40:59.199><c> bits</c><01:41:00.040><c> um</c><01:41:00.320><c> generally</c>"
+    },
+    {
+      "start": 6060.87,
+      "duration": 0.0,
+      "text": "are in 60 bit in 16 bits um generally"
+    },
+    {
+      "start": 6060.88,
+      "duration": 0.0,
+      "text": "are in 60 bit in 16 bits um generally the<01:41:01.000><c> way</c><01:41:01.159><c> you</c><01:41:01.280><c> should</c><01:41:01.440><c> be</c><01:41:01.560><c> thinking</c><01:41:01.800><c> about</c><01:41:02.000><c> it</c>"
+    },
+    {
+      "start": 6062.109,
+      "duration": 0.0,
+      "text": "the way you should be thinking about it"
+    },
+    {
+      "start": 6062.119,
+      "duration": 0.0,
+      "text": "the way you should be thinking about it is<01:41:02.199><c> that</c><01:41:02.599><c> your</c><01:41:02.800><c> weights</c><01:41:03.639><c> are</c><01:41:03.880><c> stored</c><01:41:04.360><c> of</c><01:41:04.480><c> your</c>"
+    },
+    {
+      "start": 6064.589,
+      "duration": 0.0,
+      "text": "is that your weights are stored of your"
+    },
+    {
+      "start": 6064.599,
+      "duration": 0.0,
+      "text": "is that your weights are stored of your model<01:41:04.840><c> are</c><01:41:04.960><c> stored</c><01:41:05.199><c> in</c><01:41:05.320><c> 32</c><01:41:05.760><c> bits</c><01:41:06.679><c> um</c><01:41:07.159><c> but</c><01:41:07.400><c> just</c>"
+    },
+    {
+      "start": 6067.589,
+      "duration": 0.0,
+      "text": "model are stored in 32 bits um but just"
+    },
+    {
+      "start": 6067.599,
+      "duration": 0.0,
+      "text": "model are stored in 32 bits um but just before<01:41:07.840><c> the</c><01:41:08.000><c> computation</c><01:41:08.480><c> you</c><01:41:08.599><c> put</c>"
+    },
+    {
+      "start": 6068.79,
+      "duration": 0.0,
+      "text": "before the computation you put"
+    },
+    {
+      "start": 6068.8,
+      "duration": 0.0,
+      "text": "before the computation you put everything<01:41:09.040><c> in</c><01:41:09.280><c> 16</c><01:41:09.719><c> 16</c><01:41:10.119><c> bits</c><01:41:10.520><c> like</c><01:41:10.639><c> this</c><01:41:10.760><c> you</c>"
+    },
+    {
+      "start": 6070.87,
+      "duration": 0.0,
+      "text": "everything in 16 16 bits like this you"
+    },
+    {
+      "start": 6070.88,
+      "duration": 0.0,
+      "text": "everything in 16 16 bits like this you do<01:41:11.040><c> computation</c><01:41:11.639><c> super</c><01:41:11.960><c> fast</c><01:41:12.520><c> and</c><01:41:12.679><c> at</c><01:41:12.800><c> the</c><01:41:12.960><c> end</c>"
+    },
+    {
+      "start": 6073.629,
+      "duration": 0.0,
+      "text": "do computation super fast and at the end"
+    },
+    {
+      "start": 6073.639,
+      "duration": 0.0,
+      "text": "do computation super fast and at the end you<01:41:14.080><c> update</c><01:41:14.480><c> your</c><01:41:14.599><c> weights</c><01:41:15.119><c> in</c><01:41:15.320><c> 32</c><01:41:15.800><c> Bits</c><01:41:16.239><c> And</c>"
+    },
+    {
+      "start": 6076.31,
+      "duration": 0.0,
+      "text": "you update your weights in 32 Bits And"
+    },
+    {
+      "start": 6076.32,
+      "duration": 0.0,
+      "text": "you update your weights in 32 Bits And the<01:41:16.440><c> reason</c><01:41:16.679><c> why</c><01:41:16.800><c> you</c><01:41:16.920><c> do</c><01:41:17.040><c> all</c><01:41:17.199><c> the</c><01:41:17.360><c> updates</c><01:41:17.639><c> in</c>"
+    },
+    {
+      "start": 6077.75,
+      "duration": 0.0,
+      "text": "the reason why you do all the updates in"
+    },
+    {
+      "start": 6077.76,
+      "duration": 0.0,
+      "text": "the reason why you do all the updates in 32<01:41:18.159><c> bits</c><01:41:18.599><c> it's</c><01:41:18.760><c> just</c><01:41:19.000><c> think</c><01:41:19.320><c> that</c><01:41:19.400><c> if</c><01:41:19.520><c> your</c>"
+    },
+    {
+      "start": 6079.589,
+      "duration": 0.0,
+      "text": "32 bits it's just think that if your"
+    },
+    {
+      "start": 6079.599,
+      "duration": 0.0,
+      "text": "32 bits it's just think that if your learning<01:41:19.880><c> rate</c><01:41:20.080><c> for</c><01:41:20.199><c> example</c><01:41:20.480><c> is</c><01:41:20.639><c> very</c><01:41:20.840><c> small</c>"
+    },
+    {
+      "start": 6081.43,
+      "duration": 0.0,
+      "text": "learning rate for example is very small"
+    },
+    {
+      "start": 6081.44,
+      "duration": 0.0,
+      "text": "learning rate for example is very small you<01:41:21.599><c> still</c><01:41:21.800><c> want</c><01:41:21.920><c> to</c><01:41:22.080><c> be</c><01:41:22.239><c> able</c><01:41:22.480><c> to</c><01:41:22.719><c> like</c><01:41:23.000><c> make</c><01:41:23.400><c> a</c>"
+    },
+    {
+      "start": 6083.55,
+      "duration": 0.0,
+      "text": "you still want to be able to like make a"
+    },
+    {
+      "start": 6083.56,
+      "duration": 0.0,
+      "text": "you still want to be able to like make a difference<01:41:23.960><c> in</c><01:41:24.080><c> your</c><01:41:24.280><c> weights</c><01:41:25.040><c> uh</c><01:41:25.159><c> so</c><01:41:25.280><c> all</c><01:41:25.400><c> the</c>"
+    },
+    {
+      "start": 6085.55,
+      "duration": 0.0,
+      "text": "difference in your weights uh so all the"
+    },
+    {
+      "start": 6085.56,
+      "duration": 0.0,
+      "text": "difference in your weights uh so all the computation<01:41:26.440><c> is</c><01:41:26.719><c> done</c><01:41:27.199><c> in</c><01:41:27.360><c> 16</c><01:41:27.760><c> bits</c><01:41:28.400><c> but</c><01:41:28.800><c> the</c>"
+    },
+    {
+      "start": 6088.91,
+      "duration": 0.0,
+      "text": "computation is done in 16 bits but the"
+    },
+    {
+      "start": 6088.92,
+      "duration": 0.0,
+      "text": "computation is done in 16 bits but the weights<01:41:29.159><c> are</c><01:41:29.280><c> actually</c><01:41:29.440><c> stored</c><01:41:29.760><c> in</c><01:41:29.880><c> 32</c><01:41:30.320><c> bits</c>"
+    },
+    {
+      "start": 6090.709,
+      "duration": 0.0,
+      "text": "weights are actually stored in 32 bits"
+    },
+    {
+      "start": 6090.719,
+      "duration": 0.0,
+      "text": "weights are actually stored in 32 bits so<01:41:30.880><c> that's</c><01:41:31.119><c> like</c><01:41:31.280><c> the</c><01:41:31.400><c> standard</c><01:41:31.800><c> way</c><01:41:31.960><c> that</c>"
+    },
+    {
+      "start": 6092.07,
+      "duration": 0.0,
+      "text": "so that's like the standard way that"
+    },
+    {
+      "start": 6092.08,
+      "duration": 0.0,
+      "text": "so that's like the standard way that people<01:41:32.280><c> are</c><01:41:32.400><c> doing</c><01:41:33.000><c> it</c><01:41:34.199><c> um</c><01:41:35.199><c> okay</c><01:41:35.400><c> I'll</c>"
+    },
+    {
+      "start": 6095.589,
+      "duration": 0.0,
+      "text": "people are doing it um okay I'll"
+    },
+    {
+      "start": 6095.599,
+      "duration": 0.0,
+      "text": "people are doing it um okay I'll actually<01:41:35.880><c> talk</c><01:41:36.280><c> just</c><01:41:36.440><c> about</c><01:41:36.679><c> this</c><01:41:36.800><c> and</c><01:41:36.920><c> then</c>"
+    },
+    {
+      "start": 6096.99,
+      "duration": 0.0,
+      "text": "actually talk just about this and then"
+    },
+    {
+      "start": 6097.0,
+      "duration": 0.0,
+      "text": "actually talk just about this and then I'll<01:41:37.159><c> skip</c><01:41:37.360><c> all</c><01:41:37.480><c> the</c><01:41:37.599><c> rest</c><01:41:37.840><c> operator</c><01:41:38.280><c> Fusion</c>"
+    },
+    {
+      "start": 6098.589,
+      "duration": 0.0,
+      "text": "I'll skip all the rest operator Fusion"
+    },
+    {
+      "start": 6098.599,
+      "duration": 0.0,
+      "text": "I'll skip all the rest operator Fusion because<01:41:38.719><c> I</c><01:41:38.800><c> think</c><01:41:38.920><c> this</c><01:41:39.000><c> is</c><01:41:39.119><c> actually</c><01:41:39.400><c> pretty</c>"
+    },
+    {
+      "start": 6099.589,
+      "duration": 0.0,
+      "text": "because I think this is actually pretty"
+    },
+    {
+      "start": 6099.599,
+      "duration": 0.0,
+      "text": "because I think this is actually pretty cool<01:41:40.400><c> as</c><01:41:40.520><c> I</c><01:41:40.639><c> just</c><01:41:40.800><c> said</c><01:41:41.040><c> communication</c><01:41:41.599><c> is</c>"
+    },
+    {
+      "start": 6101.79,
+      "duration": 0.0,
+      "text": "cool as I just said communication is"
+    },
+    {
+      "start": 6101.8,
+      "duration": 0.0,
+      "text": "cool as I just said communication is very<01:41:42.040><c> slow</c><01:41:42.880><c> and</c><01:41:43.080><c> actually</c><01:41:43.520><c> every</c><01:41:43.800><c> time</c><01:41:44.199><c> you</c>"
+    },
+    {
+      "start": 6104.35,
+      "duration": 0.0,
+      "text": "very slow and actually every time you"
+    },
+    {
+      "start": 6104.36,
+      "duration": 0.0,
+      "text": "very slow and actually every time you use<01:41:44.560><c> a</c><01:41:44.760><c> pie</c><01:41:44.960><c> torch</c><01:41:45.239><c> line</c><01:41:45.920><c> it</c><01:41:46.119><c> basically</c><01:41:46.480><c> moves</c>"
+    },
+    {
+      "start": 6106.87,
+      "duration": 0.0,
+      "text": "use a pie torch line it basically moves"
+    },
+    {
+      "start": 6106.88,
+      "duration": 0.0,
+      "text": "use a pie torch line it basically moves variable<01:41:47.280><c> to</c><01:41:47.440><c> Global</c><01:41:47.760><c> memory</c><01:41:48.040><c> of</c><01:41:48.159><c> your</c><01:41:48.320><c> GPU</c><01:41:49.040><c> so</c>"
+    },
+    {
+      "start": 6109.189,
+      "duration": 0.0,
+      "text": "variable to Global memory of your GPU so"
+    },
+    {
+      "start": 6109.199,
+      "duration": 0.0,
+      "text": "variable to Global memory of your GPU so when<01:41:49.320><c> you</c><01:41:49.440><c> have</c><01:41:49.639><c> something</c><01:41:49.920><c> like</c><01:41:50.159><c> this</c><01:41:50.760><c> x</c><01:41:51.360><c> do</c>"
+    },
+    {
+      "start": 6112.149,
+      "duration": 0.0,
+      "text": "when you have something like this x do"
+    },
+    {
+      "start": 6112.159,
+      "duration": 0.0,
+      "text": "when you have something like this x do cosine<01:41:53.400><c> uh</c><01:41:53.520><c> equal</c><01:41:53.880><c> X1</c><01:41:54.360><c> and</c><01:41:54.480><c> then</c><01:41:54.599><c> you</c><01:41:55.080><c> do</c><01:41:55.239><c> X1</c><01:41:55.679><c> do</c>"
+    },
+    {
+      "start": 6115.87,
+      "duration": 0.0,
+      "text": "cosine uh equal X1 and then you do X1 do"
+    },
+    {
+      "start": 6115.88,
+      "duration": 0.0,
+      "text": "cosine uh equal X1 and then you do X1 do cosine<01:41:56.440><c> what</c><01:41:56.560><c> is</c><01:41:56.719><c> happening</c><01:41:57.159><c> behind</c><01:41:57.480><c> the</c>"
+    },
+    {
+      "start": 6117.629,
+      "duration": 0.0,
+      "text": "cosine what is happening behind the"
+    },
+    {
+      "start": 6117.639,
+      "duration": 0.0,
+      "text": "cosine what is happening behind the scenes<01:41:58.280><c> is</c><01:41:58.400><c> that</c><01:41:58.560><c> you</c><01:41:58.679><c> take</c><01:41:58.880><c> the</c><01:41:59.080><c> X</c><01:41:59.320><c> which</c><01:41:59.440><c> is</c>"
+    },
+    {
+      "start": 6119.589,
+      "duration": 0.0,
+      "text": "scenes is that you take the X which is"
+    },
+    {
+      "start": 6119.599,
+      "duration": 0.0,
+      "text": "scenes is that you take the X which is data<01:42:00.199><c> you</c><01:42:00.360><c> ship</c><01:42:00.639><c> it</c><01:42:00.800><c> to</c><01:42:01.000><c> your</c><01:42:01.719><c> um</c><01:42:01.880><c> to</c><01:42:02.040><c> your</c>"
+    },
+    {
+      "start": 6122.189,
+      "duration": 0.0,
+      "text": "data you ship it to your um to your"
+    },
+    {
+      "start": 6122.199,
+      "duration": 0.0,
+      "text": "data you ship it to your um to your actual<01:42:02.639><c> processes</c><01:42:03.080><c> of</c><01:42:03.239><c> your</c><01:42:03.360><c> gpus</c><01:42:03.960><c> you</c><01:42:04.119><c> apply</c>"
+    },
+    {
+      "start": 6124.39,
+      "duration": 0.0,
+      "text": "actual processes of your gpus you apply"
+    },
+    {
+      "start": 6124.4,
+      "duration": 0.0,
+      "text": "actual processes of your gpus you apply the<01:42:04.560><c> coign</c><01:42:05.119><c> you</c><01:42:05.239><c> ship</c><01:42:05.480><c> it</c><01:42:05.679><c> back</c><01:42:05.800><c> to</c><01:42:05.920><c> the</c><01:42:06.040><c> main</c>"
+    },
+    {
+      "start": 6126.229,
+      "duration": 0.0,
+      "text": "the coign you ship it back to the main"
+    },
+    {
+      "start": 6126.239,
+      "duration": 0.0,
+      "text": "the coign you ship it back to the main memory<01:42:06.639><c> of</c><01:42:06.719><c> your</c><01:42:06.880><c> GPU</c><01:42:07.719><c> and</c><01:42:07.880><c> then</c><01:42:08.080><c> you</c><01:42:08.280><c> see</c><01:42:08.560><c> the</c>"
+    },
+    {
+      "start": 6128.709,
+      "duration": 0.0,
+      "text": "memory of your GPU and then you see the"
+    },
+    {
+      "start": 6128.719,
+      "duration": 0.0,
+      "text": "memory of your GPU and then you see the next<01:42:08.920><c> sign</c><01:42:09.320><c> you</c><01:42:09.400><c> ship</c><01:42:09.639><c> it</c><01:42:09.840><c> back</c><01:42:09.960><c> to</c><01:42:10.119><c> the</c>"
+    },
+    {
+      "start": 6130.229,
+      "duration": 0.0,
+      "text": "next sign you ship it back to the"
+    },
+    {
+      "start": 6130.239,
+      "duration": 0.0,
+      "text": "next sign you ship it back to the computer<01:42:10.800><c> to</c><01:42:11.119><c> the</c><01:42:11.440><c> GPU</c><01:42:11.920><c> processor</c><01:42:12.480><c> you</c><01:42:12.639><c> apply</c>"
+    },
+    {
+      "start": 6132.91,
+      "duration": 0.0,
+      "text": "computer to the GPU processor you apply"
+    },
+    {
+      "start": 6132.92,
+      "duration": 0.0,
+      "text": "computer to the GPU processor you apply another<01:42:13.159><c> cosign</c><01:42:13.760><c> and</c><01:42:13.840><c> you</c><01:42:13.920><c> ship</c><01:42:14.159><c> it</c><01:42:14.280><c> back</c>"
+    },
+    {
+      "start": 6134.51,
+      "duration": 0.0,
+      "text": "another cosign and you ship it back"
+    },
+    {
+      "start": 6134.52,
+      "duration": 0.0,
+      "text": "another cosign and you ship it back again<01:42:15.440><c> um</c><01:42:15.960><c> so</c><01:42:16.239><c> another</c><01:42:16.520><c> way</c><01:42:16.639><c> to</c><01:42:16.760><c> see</c><01:42:17.000><c> that</c><01:42:17.199><c> is</c>"
+    },
+    {
+      "start": 6137.31,
+      "duration": 0.0,
+      "text": "again um so another way to see that is"
+    },
+    {
+      "start": 6137.32,
+      "duration": 0.0,
+      "text": "again um so another way to see that is that<01:42:17.440><c> you</c><01:42:17.639><c> go</c><01:42:17.800><c> from</c><01:42:17.960><c> your</c><01:42:18.159><c> Dam</c><01:42:18.639><c> which</c><01:42:18.719><c> is</c><01:42:18.800><c> your</c>"
+    },
+    {
+      "start": 6138.95,
+      "duration": 0.0,
+      "text": "that you go from your Dam which is your"
+    },
+    {
+      "start": 6138.96,
+      "duration": 0.0,
+      "text": "that you go from your Dam which is your Global<01:42:19.280><c> memory</c><01:42:19.880><c> in</c><01:42:20.000><c> your</c><01:42:20.159><c> GPU</c><01:42:20.960><c> and</c><01:42:21.080><c> you</c><01:42:21.239><c> ship</c>"
+    },
+    {
+      "start": 6141.51,
+      "duration": 0.0,
+      "text": "Global memory in your GPU and you ship"
+    },
+    {
+      "start": 6141.52,
+      "duration": 0.0,
+      "text": "Global memory in your GPU and you ship it<01:42:21.639><c> to</c><01:42:21.880><c> compute</c><01:42:22.400><c> you</c><01:42:22.480><c> ship</c><01:42:22.719><c> it</c><01:42:22.880><c> back</c><01:42:23.199><c> for</c><01:42:23.400><c> every</c>"
+    },
+    {
+      "start": 6143.629,
+      "duration": 0.0,
+      "text": "it to compute you ship it back for every"
+    },
+    {
+      "start": 6143.639,
+      "duration": 0.0,
+      "text": "it to compute you ship it back for every line<01:42:24.119><c> This</c><01:42:24.239><c> is</c><01:42:24.320><c> a</c><01:42:24.520><c> naive</c><01:42:24.880><c> way</c><01:42:25.000><c> of</c><01:42:25.159><c> doing</c><01:42:25.360><c> it</c>"
+    },
+    {
+      "start": 6145.95,
+      "duration": 0.0,
+      "text": "line This is a naive way of doing it"
+    },
+    {
+      "start": 6145.96,
+      "duration": 0.0,
+      "text": "line This is a naive way of doing it this<01:42:26.080><c> seems</c><01:42:26.440><c> very</c><01:42:26.880><c> wasteful</c><01:42:27.880><c> um</c><01:42:28.520><c> so</c><01:42:29.080><c> the</c><01:42:29.280><c> idea</c>"
+    },
+    {
+      "start": 6149.91,
+      "duration": 0.0,
+      "text": "this seems very wasteful um so the idea"
+    },
+    {
+      "start": 6149.92,
+      "duration": 0.0,
+      "text": "this seems very wasteful um so the idea simple<01:42:30.320><c> idea</c><01:42:30.679><c> of</c><01:42:30.880><c> operative</c><01:42:31.360><c> Fusion</c><01:42:31.880><c> is</c><01:42:32.040><c> just</c>"
+    },
+    {
+      "start": 6152.31,
+      "duration": 0.0,
+      "text": "simple idea of operative Fusion is just"
+    },
+    {
+      "start": 6152.32,
+      "duration": 0.0,
+      "text": "simple idea of operative Fusion is just communicate<01:42:33.320><c> do</c><01:42:33.520><c> all</c><01:42:33.679><c> the</c><01:42:33.800><c> computation</c><01:42:34.679><c> ship</c>"
+    },
+    {
+      "start": 6154.95,
+      "duration": 0.0,
+      "text": "communicate do all the computation ship"
+    },
+    {
+      "start": 6154.96,
+      "duration": 0.0,
+      "text": "communicate do all the computation ship it<01:42:35.119><c> back</c><01:42:35.360><c> once</c><01:42:36.199><c> and</c><01:42:36.400><c> this</c><01:42:36.520><c> is</c><01:42:36.719><c> exactly</c><01:42:37.199><c> what</c>"
+    },
+    {
+      "start": 6157.589,
+      "duration": 0.0,
+      "text": "it back once and this is exactly what"
+    },
+    {
+      "start": 6157.599,
+      "duration": 0.0,
+      "text": "it back once and this is exactly what fuse<01:42:37.960><c> kernels</c><01:42:38.480><c> are</c><01:42:39.360><c> um</c><01:42:39.560><c> so</c><01:42:39.760><c> if</c><01:42:39.840><c> you</c><01:42:40.080><c> ever</c><01:42:40.320><c> want</c>"
+    },
+    {
+      "start": 6160.47,
+      "duration": 0.0,
+      "text": "fuse kernels are um so if you ever want"
+    },
+    {
+      "start": 6160.48,
+      "duration": 0.0,
+      "text": "fuse kernels are um so if you ever want to<01:42:41.320><c> make</c><01:42:41.599><c> your</c><01:42:41.840><c> comp</c><01:42:42.520><c> your</c><01:42:43.520><c> computations</c><01:42:44.040><c> in</c>"
+    },
+    {
+      "start": 6164.229,
+      "duration": 0.0,
+      "text": "to make your comp your computations in"
+    },
+    {
+      "start": 6164.239,
+      "duration": 0.0,
+      "text": "to make your comp your computations in pytorch<01:42:44.760><c> much</c><01:42:45.000><c> faster</c><01:42:45.840><c> just</c><01:42:46.080><c> apply</c><01:42:46.400><c> torch.</c>"
+    },
+    {
+      "start": 6167.03,
+      "duration": 0.0,
+      "text": "pytorch much faster just apply torch."
+    },
+    {
+      "start": 6167.04,
+      "duration": 0.0,
+      "text": "pytorch much faster just apply torch. compile<01:42:47.920><c> on</c><01:42:48.119><c> your</c><01:42:48.320><c> model</c><01:42:48.960><c> this</c><01:42:49.080><c> is</c><01:42:49.560><c> going</c><01:42:49.679><c> to</c>"
+    },
+    {
+      "start": 6169.83,
+      "duration": 0.0,
+      "text": "compile on your model this is going to"
+    },
+    {
+      "start": 6169.84,
+      "duration": 0.0,
+      "text": "compile on your model this is going to make<01:42:50.080><c> your</c><01:42:50.239><c> model</c><01:42:50.520><c> around</c><01:42:50.840><c> two</c><01:42:51.080><c> times</c><01:42:51.400><c> faster</c>"
+    },
+    {
+      "start": 6172.03,
+      "duration": 0.0,
+      "text": "make your model around two times faster"
+    },
+    {
+      "start": 6172.04,
+      "duration": 0.0,
+      "text": "make your model around two times faster and<01:42:52.280><c> what</c><01:42:52.400><c> it</c><01:42:52.560><c> does</c><01:42:52.920><c> is</c><01:42:53.280><c> simply</c><01:42:53.560><c> that</c><01:42:53.679><c> it</c>"
+    },
+    {
+      "start": 6173.87,
+      "duration": 0.0,
+      "text": "and what it does is simply that it"
+    },
+    {
+      "start": 6173.88,
+      "duration": 0.0,
+      "text": "and what it does is simply that it rewrites<01:42:54.800><c> your</c><01:42:55.199><c> code</c><01:42:56.199><c> uh</c><01:42:56.360><c> your</c><01:42:56.599><c> P</c><01:42:56.920><c> like</c><01:42:57.040><c> your</c>"
+    },
+    {
+      "start": 6177.229,
+      "duration": 0.0,
+      "text": "rewrites your code uh your P like your"
+    },
+    {
+      "start": 6177.239,
+      "duration": 0.0,
+      "text": "rewrites your code uh your P like your py<01:42:57.440><c> torch</c><01:42:57.719><c> code</c><01:42:58.320><c> basically</c><01:42:59.159><c> in</c><01:42:59.360><c> C++</c><01:43:00.119><c> in</c><01:43:00.440><c> Cuda</c>"
+    },
+    {
+      "start": 6181.43,
+      "duration": 0.0,
+      "text": "py torch code basically in C++ in Cuda"
+    },
+    {
+      "start": 6181.44,
+      "duration": 0.0,
+      "text": "py torch code basically in C++ in Cuda uh<01:43:01.679><c> to</c><01:43:02.639><c> to</c><01:43:02.920><c> do</c><01:43:03.080><c> the</c><01:43:03.239><c> communication</c><01:43:03.679><c> only</c><01:43:03.920><c> once</c>"
+    },
+    {
+      "start": 6184.27,
+      "duration": 0.0,
+      "text": "uh to to do the communication only once"
+    },
+    {
+      "start": 6184.28,
+      "duration": 0.0,
+      "text": "uh to to do the communication only once then<01:43:04.400><c> do</c><01:43:04.560><c> all</c><01:43:04.719><c> the</c><01:43:04.920><c> operations</c><01:43:05.560><c> then</c><01:43:06.040><c> uh</c><01:43:06.159><c> ship</c>"
+    },
+    {
+      "start": 6186.39,
+      "duration": 0.0,
+      "text": "then do all the operations then uh ship"
+    },
+    {
+      "start": 6186.4,
+      "duration": 0.0,
+      "text": "then do all the operations then uh ship it<01:43:07.000><c> back</c><01:43:08.000><c> okay</c><01:43:08.239><c> I'm</c><01:43:08.400><c> not</c><01:43:08.520><c> going</c><01:43:08.639><c> to</c><01:43:08.760><c> have</c><01:43:08.920><c> time</c>"
+    },
+    {
+      "start": 6189.07,
+      "duration": 0.0,
+      "text": "it back okay I'm not going to have time"
+    },
+    {
+      "start": 6189.08,
+      "duration": 0.0,
+      "text": "it back okay I'm not going to have time to<01:43:09.239><c> talk</c><01:43:09.400><c> about</c><01:43:09.599><c> tiling</c><01:43:10.400><c> tiling</c><01:43:10.719><c> is</c><01:43:10.920><c> important</c>"
+    },
+    {
+      "start": 6191.709,
+      "duration": 0.0,
+      "text": "to talk about tiling tiling is important"
+    },
+    {
+      "start": 6191.719,
+      "duration": 0.0,
+      "text": "to talk about tiling tiling is important paration<01:43:12.639><c> paration</c><01:43:13.199><c> is</c><01:43:13.639><c> important</c><01:43:14.840><c> um</c><01:43:15.840><c> and</c>"
+    },
+    {
+      "start": 6196.109,
+      "duration": 0.0,
+      "text": "paration paration is important um and"
+    },
+    {
+      "start": 6196.119,
+      "duration": 0.0,
+      "text": "paration paration is important um and mixture<01:43:16.440><c> of</c><01:43:16.599><c> experts</c><01:43:17.159><c> mixture</c><01:43:17.440><c> of</c><01:43:17.560><c> experts</c><01:43:18.000><c> is</c>"
+    },
+    {
+      "start": 6198.189,
+      "duration": 0.0,
+      "text": "mixture of experts mixture of experts is"
+    },
+    {
+      "start": 6198.199,
+      "duration": 0.0,
+      "text": "mixture of experts mixture of experts is important<01:43:18.920><c> Outlook</c><01:43:19.880><c> there</c><01:43:20.000><c> are</c><01:43:20.119><c> many</c><01:43:20.320><c> things</c>"
+    },
+    {
+      "start": 6200.51,
+      "duration": 0.0,
+      "text": "important Outlook there are many things"
+    },
+    {
+      "start": 6200.52,
+      "duration": 0.0,
+      "text": "important Outlook there are many things we<01:43:20.639><c> haven't</c><01:43:21.000><c> T</c><01:43:22.000><c> talked</c><01:43:22.320><c> about</c><01:43:23.239><c> we</c><01:43:23.360><c> haven't</c>"
+    },
+    {
+      "start": 6203.55,
+      "duration": 0.0,
+      "text": "we haven't T talked about we haven't"
+    },
+    {
+      "start": 6203.56,
+      "duration": 0.0,
+      "text": "we haven't T talked about we haven't talked<01:43:23.800><c> about</c><01:43:24.280><c> architectures</c><01:43:24.920><c> we</c><01:43:25.080><c> definitely</c>"
+    },
+    {
+      "start": 6205.35,
+      "duration": 0.0,
+      "text": "talked about architectures we definitely"
+    },
+    {
+      "start": 6205.36,
+      "duration": 0.0,
+      "text": "talked about architectures we definitely haven't<01:43:25.599><c> talked</c><01:43:25.800><c> about</c><01:43:26.280><c> inference</c><01:43:27.280><c> um</c><01:43:27.639><c> there</c>"
+    },
+    {
+      "start": 6207.75,
+      "duration": 0.0,
+      "text": "haven't talked about inference um there"
+    },
+    {
+      "start": 6207.76,
+      "duration": 0.0,
+      "text": "haven't talked about inference um there are<01:43:27.880><c> many</c><01:43:28.119><c> other</c><01:43:28.280><c> things</c><01:43:28.440><c> that</c><01:43:28.560><c> are</c><01:43:28.719><c> important</c>"
+    },
+    {
+      "start": 6209.03,
+      "duration": 0.0,
+      "text": "are many other things that are important"
+    },
+    {
+      "start": 6209.04,
+      "duration": 0.0,
+      "text": "are many other things that are important with<01:43:29.199><c> LMS</c><01:43:30.000><c> what</c><01:43:30.119><c> is</c><01:43:30.239><c> the</c><01:43:30.400><c> UI</c><01:43:30.760><c> that</c><01:43:30.880><c> you</c><01:43:31.080><c> use</c><01:43:31.320><c> I</c>"
+    },
+    {
+      "start": 6211.39,
+      "duration": 0.0,
+      "text": "with LMS what is the UI that you use I"
+    },
+    {
+      "start": 6211.4,
+      "duration": 0.0,
+      "text": "with LMS what is the UI that you use I mean<01:43:31.880><c> arguably</c><01:43:32.360><c> chat</c><01:43:32.599><c> jpt</c><01:43:33.000><c> the</c><01:43:33.159><c> big</c><01:43:33.320><c> novelty</c>"
+    },
+    {
+      "start": 6213.709,
+      "duration": 0.0,
+      "text": "mean arguably chat jpt the big novelty"
+    },
+    {
+      "start": 6213.719,
+      "duration": 0.0,
+      "text": "mean arguably chat jpt the big novelty was<01:43:33.960><c> just</c><01:43:34.320><c> have</c><01:43:34.480><c> a</c><01:43:34.599><c> simple</c><01:43:34.880><c> UI</c><01:43:35.199><c> to</c><01:43:35.320><c> use</c><01:43:35.480><c> it</c>"
+    },
+    {
+      "start": 6215.91,
+      "duration": 0.0,
+      "text": "was just have a simple UI to use it"
+    },
+    {
+      "start": 6215.92,
+      "duration": 0.0,
+      "text": "was just have a simple UI to use it multimodality<01:43:36.880><c> what</c><01:43:37.000><c> are</c><01:43:37.119><c> all</c><01:43:37.239><c> the</c><01:43:37.360><c> misuses</c>"
+    },
+    {
+      "start": 6217.79,
+      "duration": 0.0,
+      "text": "multimodality what are all the misuses"
+    },
+    {
+      "start": 6217.8,
+      "duration": 0.0,
+      "text": "multimodality what are all the misuses you<01:43:37.920><c> could</c><01:43:38.159><c> have</c><01:43:38.840><c> uh</c><01:43:38.960><c> the</c><01:43:39.119><c> fact</c><01:43:39.280><c> that</c><01:43:39.400><c> there</c>"
+    },
+    {
+      "start": 6219.55,
+      "duration": 0.0,
+      "text": "you could have uh the fact that there"
+    },
+    {
+      "start": 6219.56,
+      "duration": 0.0,
+      "text": "you could have uh the fact that there might<01:43:39.719><c> not</c><01:43:39.840><c> be</c><01:43:40.000><c> enough</c><01:43:40.239><c> data</c><01:43:40.440><c> on</c><01:43:40.560><c> the</c><01:43:40.719><c> internet</c>"
+    },
+    {
+      "start": 6221.03,
+      "duration": 0.0,
+      "text": "might not be enough data on the internet"
+    },
+    {
+      "start": 6221.04,
+      "duration": 0.0,
+      "text": "might not be enough data on the internet to<01:43:41.159><c> train</c><01:43:41.400><c> all</c><01:43:41.560><c> these</c><01:43:41.719><c> models</c><01:43:42.440><c> legality</c><01:43:42.920><c> of</c>"
+    },
+    {
+      "start": 6223.03,
+      "duration": 0.0,
+      "text": "to train all these models legality of"
+    },
+    {
+      "start": 6223.04,
+      "duration": 0.0,
+      "text": "to train all these models legality of data<01:43:43.280><c> collection</c><01:43:43.920><c> so</c><01:43:44.119><c> many</c><01:43:44.400><c> other</c><01:43:44.639><c> things</c><01:43:45.320><c> if</c>"
+    },
+    {
+      "start": 6225.43,
+      "duration": 0.0,
+      "text": "data collection so many other things if"
+    },
+    {
+      "start": 6225.44,
+      "duration": 0.0,
+      "text": "data collection so many other things if you<01:43:45.560><c> are</c><01:43:45.760><c> interested</c><01:43:46.119><c> in</c><01:43:46.280><c> all</c><01:43:46.440><c> these</c><01:43:46.679><c> topics</c>"
+    },
+    {
+      "start": 6227.669,
+      "duration": 0.0,
+      "text": "you are interested in all these topics"
+    },
+    {
+      "start": 6227.679,
+      "duration": 0.0,
+      "text": "you are interested in all these topics uh<01:43:47.840><c> I</c><01:43:47.920><c> would</c><01:43:48.119><c> suggest</c><01:43:48.520><c> three</c><01:43:48.760><c> classes</c><01:43:49.840><c> cs224n</c>"
+    },
+    {
+      "start": 6230.83,
+      "duration": 0.0,
+      "text": "uh I would suggest three classes cs224n"
+    },
+    {
+      "start": 6230.84,
+      "duration": 0.0,
+      "text": "uh I would suggest three classes cs224n is<01:43:51.000><c> probably</c><01:43:51.280><c> the</c><01:43:51.360><c> one</c><01:43:51.480><c> that</c><01:43:51.599><c> touches</c><01:43:51.960><c> the</c>"
+    },
+    {
+      "start": 6232.109,
+      "duration": 0.0,
+      "text": "is probably the one that touches the"
+    },
+    {
+      "start": 6232.119,
+      "duration": 0.0,
+      "text": "is probably the one that touches the least<01:43:52.719><c> on</c><01:43:53.400><c> uh</c><01:43:53.760><c> LMS</c><01:43:54.760><c> uh</c><01:43:54.880><c> but</c><01:43:55.000><c> it</c><01:43:55.119><c> gives</c><01:43:55.320><c> some</c>"
+    },
+    {
+      "start": 6235.51,
+      "duration": 0.0,
+      "text": "least on uh LMS uh but it gives some"
+    },
+    {
+      "start": 6235.52,
+      "duration": 0.0,
+      "text": "least on uh LMS uh but it gives some background<01:43:55.840><c> and</c><01:43:56.040><c> historical</c><01:43:56.639><c> context</c><01:43:57.639><c> um</c><01:43:58.040><c> of</c>"
+    },
+    {
+      "start": 6238.27,
+      "duration": 0.0,
+      "text": "background and historical context um of"
+    },
+    {
+      "start": 6238.28,
+      "duration": 0.0,
+      "text": "background and historical context um of all<01:43:58.520><c> the</c><01:43:58.679><c> LMS</c><01:43:59.119><c> and</c><01:43:59.280><c> gives</c><01:43:59.719><c> kind</c><01:43:59.840><c> of</c><01:43:59.960><c> some</c>"
+    },
+    {
+      "start": 6240.229,
+      "duration": 0.0,
+      "text": "all the LMS and gives kind of some"
+    },
+    {
+      "start": 6240.239,
+      "duration": 0.0,
+      "text": "all the LMS and gives kind of some adjacent<01:44:00.800><c> material</c><01:44:01.599><c> CS</c><01:44:02.000><c> 324</c><01:44:02.840><c> I</c><01:44:02.960><c> think</c><01:44:03.119><c> it's</c>"
+    },
+    {
+      "start": 6243.35,
+      "duration": 0.0,
+      "text": "adjacent material CS 324 I think it's"
+    },
+    {
+      "start": 6243.36,
+      "duration": 0.0,
+      "text": "adjacent material CS 324 I think it's called<01:44:04.040><c> Uh</c><01:44:05.040><c> I</c><01:44:05.159><c> think</c><01:44:05.280><c> it's</c><01:44:05.360><c> just</c><01:44:05.520><c> called</c><01:44:05.840><c> large</c>"
+    },
+    {
+      "start": 6246.07,
+      "duration": 0.0,
+      "text": "called Uh I think it's just called large"
+    },
+    {
+      "start": 6246.08,
+      "duration": 0.0,
+      "text": "called Uh I think it's just called large language<01:44:06.360><c> models</c><01:44:07.199><c> uh</c><01:44:07.360><c> more</c><01:44:07.599><c> in-depth</c><01:44:08.000><c> reading</c>"
+    },
+    {
+      "start": 6248.229,
+      "duration": 0.0,
+      "text": "language models uh more in-depth reading"
+    },
+    {
+      "start": 6248.239,
+      "duration": 0.0,
+      "text": "language models uh more in-depth reading and<01:44:08.440><c> lectures</c><01:44:08.920><c> on</c><01:44:09.239><c> everything</c><01:44:09.520><c> I</c><01:44:09.679><c> talked</c>"
+    },
+    {
+      "start": 6249.91,
+      "duration": 0.0,
+      "text": "and lectures on everything I talked"
+    },
+    {
+      "start": 6249.92,
+      "duration": 0.0,
+      "text": "and lectures on everything I talked about<01:44:10.280><c> CS</c><01:44:10.880><c> 336</c><01:44:11.880><c> which</c><01:44:12.000><c> is</c><01:44:12.280><c> large</c><01:44:12.560><c> language</c>"
+    },
+    {
+      "start": 6252.87,
+      "duration": 0.0,
+      "text": "about CS 336 which is large language"
+    },
+    {
+      "start": 6252.88,
+      "duration": 0.0,
+      "text": "about CS 336 which is large language model<01:44:13.199><c> from</c><01:44:13.400><c> scratch</c><01:44:13.960><c> you</c><01:44:14.280><c> actually</c><01:44:14.560><c> build</c>"
+    },
+    {
+      "start": 6254.83,
+      "duration": 0.0,
+      "text": "model from scratch you actually build"
+    },
+    {
+      "start": 6254.84,
+      "duration": 0.0,
+      "text": "model from scratch you actually build your<01:44:15.000><c> own</c><01:44:15.679><c> llm</c><01:44:16.679><c> uh</c><01:44:16.960><c> it's</c><01:44:17.320><c> an</c><01:44:17.480><c> amazing</c><01:44:17.960><c> class</c>"
+    },
+    {
+      "start": 6258.47,
+      "duration": 0.0,
+      "text": "your own llm uh it's an amazing class"
+    },
+    {
+      "start": 6258.48,
+      "duration": 0.0,
+      "text": "your own llm uh it's an amazing class also<01:44:18.719><c> given</c><01:44:19.040><c> by</c><01:44:19.239><c> my</c><01:44:19.360><c> two</c><01:44:19.679><c> supervisors</c><01:44:20.639><c> very</c>"
+    },
+    {
+      "start": 6260.91,
+      "duration": 0.0,
+      "text": "also given by my two supervisors very"
+    },
+    {
+      "start": 6260.92,
+      "duration": 0.0,
+      "text": "also given by my two supervisors very heavy<01:44:21.159><c> workload</c><01:44:21.639><c> so</c><01:44:21.800><c> be</c><01:44:21.920><c> careful</c><01:44:22.920><c> and</c><01:44:23.239><c> um</c>"
+    },
+    {
+      "start": 6263.95,
+      "duration": 0.0,
+      "text": "heavy workload so be careful and um"
+    },
+    {
+      "start": 6263.96,
+      "duration": 0.0,
+      "text": "heavy workload so be careful and um great"
+    }
+  ],
+  "plain": "so<00:00:05.879><c> let's</c><00:00:06.120><c> get</c><00:00:06.279><c> started</c><00:00:06.960><c> uh</c><00:00:07.080><c> so</c><00:00:07.200><c> I'll</c><00:00:07.359><c> be</c>\nso let's get started uh so I'll be\nso let's get started uh so I'll be talking<00:00:07.799><c> about</c><00:00:08.080><c> building</c><00:00:08.599><c> llms</c><00:00:09.200><c> today</c><00:00:09.920><c> um</c><00:00:10.120><c> so</c>\ntalking about building llms today um so\ntalking about building llms today um so I<00:00:10.400><c> think</c><00:00:10.559><c> a</c><00:00:10.679><c> lot</c><00:00:10.800><c> of</c><00:00:10.960><c> you</c><00:00:11.240><c> have</c><00:00:11.440><c> heard</c><00:00:12.160><c> of</c><00:00:12.320><c> llms</c>\nI think a lot of you have heard of llms\nI think a lot of you have heard of llms before<00:00:13.920><c> uh</c><00:00:14.080><c> but</c><00:00:14.240><c> just</c><00:00:14.400><c> as</c><00:00:14.519><c> a</c><00:00:14.719><c> quick</c><00:00:15.000><c> recap</c><00:00:15.920><c> uh</c>\nbefore uh but just as a quick recap uh\nbefore uh but just as a quick recap uh llms<00:00:16.760><c> standing</c><00:00:17.080><c> for</c><00:00:17.359><c> large</c><00:00:17.680><c> language</c><00:00:18.000><c> models</c>\nllms standing for large language models\nllms standing for large language models are<00:00:18.920><c> basically</c><00:00:19.279><c> all</c><00:00:19.439><c> the</c><00:00:19.560><c> chat</c><00:00:19.840><c> Bots</c><00:00:20.680><c> uh</c><00:00:20.840><c> that</c>\nare basically all the chat Bots uh that\nare basically all the chat Bots uh that you've<00:00:21.119><c> been</c><00:00:21.320><c> hearing</c><00:00:21.920><c> about</c><00:00:22.279><c> recently</c><00:00:22.880><c> so</c><00:00:23.800><c> uh</c>\nyou've been hearing about recently so uh\nyou've been hearing about recently so uh Chad<00:00:24.240><c> GPT</c><00:00:25.039><c> from</c><00:00:25.240><c> open</c><00:00:25.640><c> ey</c><00:00:26.320><c> Claud</c><00:00:27.199><c> from</c>\nChad GPT from open ey Claud from\nChad GPT from open ey Claud from entropic<00:00:28.039><c> Gemini</c><00:00:28.760><c> and</c><00:00:28.960><c> and</c><00:00:29.080><c> lman</c><00:00:29.679><c> other</c><00:00:30.039><c> type</c>\nentropic Gemini and and lman other type\nentropic Gemini and and lman other type of<00:00:30.279><c> models</c><00:00:30.599><c> like</c><00:00:30.800><c> this</c><00:00:31.400><c> and</c><00:00:31.519><c> today</c><00:00:31.759><c> we'll</c><00:00:31.960><c> be</c>\nof models like this and today we'll be\nof models like this and today we'll be talking<00:00:32.439><c> about</c><00:00:32.800><c> how</c><00:00:33.079><c> do</c><00:00:33.399><c> they</c><00:00:33.719><c> actually</c><00:00:34.040><c> work</c>\ntalking about how do they actually work\ntalking about how do they actually work so<00:00:34.760><c> it's</c><00:00:34.879><c> going</c><00:00:35.000><c> to</c><00:00:35.040><c> be</c><00:00:35.160><c> an</c><00:00:35.280><c> overview</c><00:00:35.680><c> because</c>\nso it's going to be an overview because\nso it's going to be an overview because it's<00:00:35.960><c> only</c><00:00:36.200><c> one</c><00:00:36.399><c> lecture</c><00:00:36.840><c> and</c><00:00:36.920><c> it's</c><00:00:37.079><c> hard</c><00:00:37.239><c> to</c>\nit's only one lecture and it's hard to\nit's only one lecture and it's hard to compress<00:00:37.760><c> everything</c><00:00:38.239><c> but</c><00:00:38.680><c> hopefully</c><00:00:39.120><c> I'll</c>\ncompress everything but hopefully I'll\ncompress everything but hopefully I'll touch<00:00:39.559><c> a</c><00:00:39.640><c> little</c><00:00:39.800><c> bit</c><00:00:40.000><c> about</c><00:00:40.280><c> all</c><00:00:40.440><c> the</c>\ntouch a little bit about all the\ntouch a little bit about all the components<00:00:40.960><c> that</c><00:00:41.039><c> are</c><00:00:41.160><c> needed</c><00:00:41.399><c> to</c><00:00:41.520><c> train</c><00:00:42.280><c> uh</c>\ncomponents that are needed to train uh\ncomponents that are needed to train uh some<00:00:42.559><c> of</c><00:00:42.719><c> these</c><00:00:43.000><c> llms</c><00:00:44.000><c> uh</c><00:00:44.120><c> also</c><00:00:44.360><c> if</c><00:00:44.480><c> you</c><00:00:44.600><c> have</c>\nsome of these llms uh also if you have\nsome of these llms uh also if you have questions<00:00:45.320><c> please</c><00:00:45.640><c> interrupt</c><00:00:46.120><c> me</c><00:00:46.399><c> and</c><00:00:46.680><c> ask</c><00:00:47.600><c> uh</c>\nquestions please interrupt me and ask uh\nquestions please interrupt me and ask uh if<00:00:47.879><c> you</c><00:00:48.039><c> have</c><00:00:48.160><c> a</c><00:00:48.360><c> question</c><00:00:48.920><c> most</c><00:00:49.239><c> likely</c><00:00:49.600><c> other</c>\nif you have a question most likely other\nif you have a question most likely other people<00:00:50.039><c> in</c><00:00:50.160><c> the</c><00:00:50.280><c> room</c><00:00:50.840><c> or</c><00:00:51.120><c> on</c><00:00:51.360><c> Zoom</c><00:00:52.120><c> have</c><00:00:52.359><c> other</c>\npeople in the room or on Zoom have other\npeople in the room or on Zoom have other have<00:00:52.960><c> the</c><00:00:53.079><c> same</c><00:00:53.320><c> question</c><00:00:53.719><c> so</c><00:00:54.079><c> please</c><00:00:54.800><c> ask</c><00:00:55.800><c> um</c>\nhave the same question so please ask um\nhave the same question so please ask um great<00:00:56.920><c> so</c><00:00:57.160><c> what</c><00:00:57.399><c> matters</c><00:00:57.920><c> when</c><00:00:58.079><c> training</c><00:00:58.760><c> llms</c>\ngreat so what matters when training llms\ngreat so what matters when training llms um<00:01:00.160><c> so</c><00:01:00.320><c> there</c><00:01:00.440><c> a</c><00:01:00.519><c> few</c><00:01:00.760><c> key</c><00:01:00.960><c> components</c><00:01:01.680><c> that</c>\num so there a few key components that\num so there a few key components that matter<00:01:02.879><c> uh</c><00:01:03.000><c> one</c><00:01:03.199><c> is</c><00:01:03.359><c> the</c><00:01:03.480><c> architecture</c><00:01:04.040><c> so</c><00:01:04.239><c> as</c>\nmatter uh one is the architecture so as\nmatter uh one is the architecture so as you<00:01:04.640><c> probably</c><00:01:04.960><c> all</c><00:01:05.119><c> know</c><00:01:05.479><c> LMS</c><00:01:06.040><c> are</c><00:01:06.240><c> newal</c>\nyou probably all know LMS are newal\nyou probably all know LMS are newal networks<00:01:07.520><c> and</c><00:01:07.759><c> when</c><00:01:08.000><c> you</c><00:01:08.240><c> think</c><00:01:08.439><c> about</c><00:01:08.640><c> new</c>\nnetworks and when you think about new\nnetworks and when you think about new networks<00:01:09.240><c> you</c><00:01:09.360><c> have</c><00:01:09.479><c> to</c><00:01:09.640><c> think</c><00:01:09.799><c> about</c><00:01:10.080><c> what</c>\nnetworks you have to think about what\nnetworks you have to think about what architecture<00:01:10.720><c> you're</c><00:01:10.880><c> using</c><00:01:11.720><c> and</c><00:01:11.880><c> another</c>\narchitecture you're using and another\narchitecture you're using and another component<00:01:12.520><c> which</c><00:01:12.640><c> is</c><00:01:12.759><c> really</c><00:01:13.000><c> important</c><00:01:13.840><c> uh</c>\ncomponent which is really important uh\ncomponent which is really important uh is<00:01:14.000><c> the</c><00:01:14.159><c> training</c><00:01:14.439><c> loss</c><00:01:14.799><c> and</c><00:01:14.880><c> the</c><00:01:15.000><c> training</c>\nis the training loss and the training\nis the training loss and the training algorithm<00:01:16.720><c> um</c><00:01:17.080><c> so</c><00:01:17.360><c> how</c><00:01:17.560><c> you</c><00:01:17.799><c> actually</c><00:01:18.320><c> train</c>\nalgorithm um so how you actually train\nalgorithm um so how you actually train these<00:01:18.840><c> models</c><00:01:19.640><c> then</c><00:01:19.759><c> it's</c><00:01:20.040><c> data</c><00:01:20.680><c> so</c><00:01:21.560><c> uh</c><00:01:21.759><c> what</c>\nthese models then it's data so uh what\nthese models then it's data so uh what do<00:01:22.000><c> you</c><00:01:22.159><c> train</c><00:01:22.439><c> these</c><00:01:22.640><c> models</c><00:01:23.159><c> on</c><00:01:24.159><c> um</c><00:01:24.520><c> the</c>\ndo you train these models on um the\ndo you train these models on um the evaluation<00:01:25.520><c> which</c><00:01:25.640><c> is</c><00:01:25.799><c> how</c><00:01:25.960><c> do</c><00:01:26.040><c> you</c><00:01:26.200><c> know</c>\nevaluation which is how do you know\nevaluation which is how do you know whether<00:01:26.600><c> you're</c><00:01:26.880><c> actually</c><00:01:27.119><c> making</c><00:01:27.920><c> progress</c>\nwhether you're actually making progress\nwhether you're actually making progress towards<00:01:29.000><c> the</c><00:01:29.159><c> goal</c><00:01:29.720><c> of</c><00:01:29.960><c> of</c><00:01:30.400><c> uh</c><00:01:30.920><c> llms</c><00:01:31.920><c> and</c><00:01:32.079><c> then</c>\ntowards the goal of of uh llms and then\ntowards the goal of of uh llms and then the<00:01:32.439><c> system</c><00:01:32.799><c> component</c><00:01:33.280><c> so</c><00:01:33.479><c> that</c><00:01:33.560><c> is</c><00:01:33.759><c> like</c><00:01:34.079><c> how</c>\nthe system component so that is like how\nthe system component so that is like how do<00:01:34.360><c> you</c><00:01:34.720><c> actually</c><00:01:35.119><c> make</c><00:01:35.360><c> these</c><00:01:35.560><c> models</c><00:01:36.000><c> run</c><00:01:36.640><c> on</c>\ndo you actually make these models run on\ndo you actually make these models run on uh<00:01:37.439><c> Modern</c><00:01:37.880><c> Hardware</c><00:01:38.439><c> which</c><00:01:38.560><c> is</c><00:01:38.680><c> really</c>\nuh Modern Hardware which is really\nuh Modern Hardware which is really important<00:01:39.240><c> because</c><00:01:39.399><c> these</c><00:01:39.520><c> models</c><00:01:39.759><c> are</c>\nimportant because these models are\nimportant because these models are really<00:01:40.159><c> large</c><00:01:40.960><c> um</c><00:01:41.159><c> so</c><00:01:41.759><c> now</c><00:01:42.000><c> more</c><00:01:42.200><c> than</c><00:01:42.399><c> ever</c>\nreally large um so now more than ever\nreally large um so now more than ever system<00:01:43.119><c> is</c><00:01:43.320><c> actually</c><00:01:43.560><c> really</c><00:01:43.799><c> an</c><00:01:44.000><c> important</c>\nsystem is actually really an important\nsystem is actually really an important topic<00:01:45.280><c> um</c><00:01:45.680><c> for</c>\ntopic um for\ntopic um for llms<00:01:47.479><c> so</c><00:01:48.079><c> those</c><00:01:48.360><c> five</c><00:01:48.600><c> components</c><00:01:49.520><c> um</c><00:01:49.880><c> You</c>\nllms so those five components um You\nllms so those five components um You probably<00:01:50.439><c> all</c><00:01:50.640><c> know</c><00:01:50.960><c> that</c><00:01:51.360><c> llms</c><00:01:52.119><c> and</c><00:01:52.240><c> if</c><00:01:52.360><c> you</c>\nprobably all know that llms and if you\nprobably all know that llms and if you don't<00:01:52.640><c> know</c><00:01:53.000><c> LMS</c><00:01:53.439><c> are</c><00:01:53.560><c> all</c><00:01:53.759><c> based</c><00:01:54.040><c> on</c>\ndon't know LMS are all based on\ndon't know LMS are all based on Transformers<00:01:54.840><c> or</c><00:01:55.000><c> at</c><00:01:55.119><c> least</c><00:01:55.360><c> some</c><00:01:55.600><c> version</c><00:01:55.880><c> of</c>\nTransformers or at least some version of\nTransformers or at least some version of Transformers<00:01:57.399><c> uh</c><00:01:57.520><c> I'm</c><00:01:57.719><c> actually</c><00:01:58.000><c> not</c><00:01:58.159><c> going</c>\nTransformers uh I'm actually not going\nTransformers uh I'm actually not going to<00:01:58.680><c> talk</c><00:01:59.119><c> about</c><00:01:59.439><c> the</c><00:01:59.520><c> AR</c><00:01:59.920><c> lecture</c><00:02:00.200><c> today</c><00:02:00.840><c> uh</c>\nto talk about the AR lecture today uh\nto talk about the AR lecture today uh one<00:02:01.280><c> because</c><00:02:01.479><c> I</c><00:02:01.600><c> gave</c><00:02:01.719><c> a</c><00:02:01.880><c> SE</c><00:02:02.240><c> lecture</c><00:02:02.960><c> on</c><00:02:03.719><c> um</c>\none because I gave a SE lecture on um\none because I gave a SE lecture on um Transformers<00:02:05.159><c> a</c><00:02:05.280><c> few</c><00:02:05.479><c> weeks</c><00:02:05.719><c> ago</c><00:02:06.560><c> and</c><00:02:06.719><c> two</c>\nTransformers a few weeks ago and two\nTransformers a few weeks ago and two because<00:02:07.159><c> you</c><00:02:07.240><c> can</c><00:02:07.399><c> find</c><00:02:07.560><c> so</c><00:02:07.880><c> much</c><00:02:08.080><c> information</c>\nbecause you can find so much information\nbecause you can find so much information online<00:02:09.239><c> on</c><00:02:09.720><c> uh</c><00:02:09.879><c> Transformers</c><00:02:10.599><c> but</c><00:02:10.679><c> I</c><00:02:10.800><c> think</c>\nonline on uh Transformers but I think\nonline on uh Transformers but I think you<00:02:11.039><c> can</c><00:02:11.440><c> it's</c><00:02:12.239><c> there's</c><00:02:12.440><c> much</c><00:02:12.680><c> less</c>\nyou can it's there's much less\nyou can it's there's much less information<00:02:13.319><c> about</c><00:02:13.560><c> the</c><00:02:13.680><c> other</c><00:02:13.879><c> four</c><00:02:14.200><c> topics</c>\ninformation about the other four topics\ninformation about the other four topics so<00:02:14.599><c> I</c><00:02:14.760><c> really</c><00:02:14.879><c> want</c><00:02:15.000><c> to</c><00:02:15.519><c> talk</c><00:02:15.760><c> about</c><00:02:16.480><c> those</c><00:02:17.480><c> um</c>\nso I really want to talk about those um\nso I really want to talk about those um another<00:02:18.120><c> thing</c><00:02:18.280><c> to</c><00:02:18.440><c> say</c><00:02:18.720><c> is</c><00:02:18.840><c> that</c><00:02:19.000><c> most</c><00:02:19.200><c> of</c>\nanother thing to say is that most of\nanother thing to say is that most of Academia<00:02:20.319><c> actually</c><00:02:20.640><c> focuses</c><00:02:21.080><c> on</c>\nAcademia actually focuses on\nAcademia actually focuses on architecture<00:02:22.239><c> and</c><00:02:22.440><c> training</c><00:02:22.879><c> algorithm</c><00:02:23.319><c> and</c>\narchitecture and training algorithm and\narchitecture and training algorithm and losses<00:02:24.519><c> um</c><00:02:25.040><c> as</c><00:02:25.239><c> academics</c><00:02:25.720><c> and</c><00:02:25.840><c> I've</c><00:02:26.000><c> done</c>\nlosses um as academics and I've done\nlosses um as academics and I've done that<00:02:26.360><c> for</c><00:02:26.519><c> a</c><00:02:26.680><c> lot</c><00:02:27.440><c> big</c><00:02:27.680><c> part</c><00:02:27.879><c> of</c><00:02:28.000><c> my</c><00:02:28.200><c> career</c><00:02:28.800><c> is</c>\nthat for a lot big part of my career is\nthat for a lot big part of my career is simply<00:02:30.040><c> we</c><00:02:30.160><c> like</c><00:02:30.480><c> thinking</c><00:02:30.879><c> that</c><00:02:31.080><c> this</c><00:02:31.200><c> is</c><00:02:31.560><c> uh</c>\nsimply we like thinking that this is uh\nsimply we like thinking that this is uh like<00:02:32.080><c> we</c><00:02:32.360><c> make</c><00:02:32.599><c> new</c><00:02:32.959><c> architectures</c><00:02:33.599><c> new</c>\nlike we make new architectures new\nlike we make new architectures new models<00:02:34.319><c> and</c><00:02:34.599><c> it</c><00:02:34.840><c> it</c><00:02:35.160><c> seems</c><00:02:35.599><c> like</c><00:02:35.760><c> it's</c><00:02:35.959><c> very</c>\nmodels and it it seems like it's very\nmodels and it it seems like it's very important<00:02:37.000><c> but</c><00:02:37.160><c> in</c><00:02:37.360><c> reality</c><00:02:37.959><c> honestly</c><00:02:38.319><c> what</c>\nimportant but in reality honestly what\nimportant but in reality honestly what matters<00:02:38.760><c> in</c><00:02:38.920><c> practice</c><00:02:39.400><c> is</c><00:02:39.560><c> mostly</c><00:02:39.920><c> the</c><00:02:40.080><c> three</c>\nmatters in practice is mostly the three\nmatters in practice is mostly the three other<00:02:41.120><c> topics</c><00:02:41.560><c> so</c><00:02:41.800><c> data</c><00:02:42.640><c> evaluation</c><00:02:43.159><c> and</c>\nother topics so data evaluation and\nother topics so data evaluation and systems<00:02:44.280><c> uh</c><00:02:44.440><c> which</c><00:02:44.560><c> is</c><00:02:44.720><c> what</c><00:02:44.920><c> of</c><00:02:45.280><c> most</c><00:02:45.480><c> of</c>\nsystems uh which is what of most of\nsystems uh which is what of most of Industry<00:02:46.400><c> actually</c><00:02:46.680><c> focuses</c><00:02:47.159><c> on</c><00:02:48.040><c> um</c><00:02:48.480><c> so</c>\nIndustry actually focuses on um so\nIndustry actually focuses on um so that's<00:02:48.840><c> also</c><00:02:49.040><c> one</c><00:02:49.159><c> of</c><00:02:49.280><c> the</c><00:02:49.360><c> reason</c><00:02:49.640><c> why</c><00:02:49.760><c> I</c>\nthat's also one of the reason why I\nthat's also one of the reason why I don't<00:02:49.959><c> want</c><00:02:50.080><c> to</c><00:02:50.200><c> talk</c><00:02:50.360><c> too</c><00:02:50.519><c> much</c><00:02:50.680><c> about</c><00:02:50.879><c> the</c>\ndon't want to talk too much about the\ndon't want to talk too much about the architecture<00:02:51.920><c> uh</c><00:02:52.080><c> because</c><00:02:52.239><c> really</c><00:02:52.440><c> the</c><00:02:52.560><c> rest</c>\narchitecture uh because really the rest\narchitecture uh because really the rest is<00:02:52.920><c> super</c>\nis super\nis super important<00:02:54.879><c> um</c><00:02:55.239><c> great</c><00:02:55.519><c> so</c><00:02:55.760><c> overview</c><00:02:56.159><c> of</c><00:02:56.280><c> the</c>\nimportant um great so overview of the\nimportant um great so overview of the lecture<00:02:57.200><c> I'll</c><00:02:57.360><c> be</c><00:02:57.519><c> talking</c><00:02:57.760><c> about</c>\nlecture I'll be talking about\nlecture I'll be talking about pre-training<00:02:58.560><c> so</c><00:02:58.760><c> pre-training</c><00:02:59.480><c> uh</c><00:02:59.560><c> you</c>\npre-training so pre-training uh you\npre-training so pre-training uh you probably<00:03:00.080><c> heard</c><00:03:00.319><c> that</c><00:03:00.480><c> word</c><00:03:00.800><c> this</c><00:03:00.879><c> is</c><00:03:01.080><c> the</c>\nprobably heard that word this is the\nprobably heard that word this is the general<00:03:01.760><c> word</c><00:03:02.200><c> this</c><00:03:02.319><c> is</c><00:03:02.480><c> kind</c><00:03:02.640><c> of</c><00:03:02.760><c> the</c>\ngeneral word this is kind of the\ngeneral word this is kind of the classical<00:03:03.720><c> language</c><00:03:04.120><c> modeling</c><00:03:05.120><c> uh</c><00:03:05.280><c> Paradigm</c>\nclassical language modeling uh Paradigm\nclassical language modeling uh Paradigm uh<00:03:06.239><c> where</c><00:03:06.360><c> you</c><00:03:06.519><c> basically</c><00:03:06.799><c> train</c><00:03:07.040><c> your</c>\nuh where you basically train your\nuh where you basically train your language<00:03:07.480><c> model</c><00:03:07.799><c> to</c><00:03:08.440><c> essentially</c><00:03:08.879><c> model</c><00:03:09.200><c> all</c>\nlanguage model to essentially model all\nlanguage model to essentially model all of<00:03:09.720><c> internet</c><00:03:10.720><c> and</c><00:03:10.879><c> then</c><00:03:11.000><c> there's</c><00:03:11.159><c> a</c><00:03:11.319><c> post</c>\nof internet and then there's a post\nof internet and then there's a post training<00:03:12.040><c> which</c><00:03:12.120><c> is</c><00:03:12.239><c> a</c><00:03:12.360><c> more</c><00:03:12.560><c> recent</c><00:03:12.879><c> Paradigm</c>\ntraining which is a more recent Paradigm\ntraining which is a more recent Paradigm which<00:03:13.480><c> is</c><00:03:13.640><c> taking</c><00:03:13.959><c> these</c><00:03:14.120><c> large</c><00:03:14.400><c> language</c>\nwhich is taking these large language\nwhich is taking these large language models<00:03:15.360><c> and</c><00:03:15.519><c> making</c><00:03:15.920><c> them</c><00:03:16.120><c> essentially</c><00:03:16.599><c> AI</c>\nmodels and making them essentially AI\nmodels and making them essentially AI assistants<00:03:18.000><c> um</c><00:03:18.159><c> so</c><00:03:18.440><c> this</c><00:03:18.560><c> is</c><00:03:18.959><c> more</c><00:03:19.159><c> of</c><00:03:19.280><c> a</c>\nassistants um so this is more of a\nassistants um so this is more of a recent<00:03:19.879><c> Trend</c><00:03:20.400><c> since</c><00:03:20.640><c> Chad</c><00:03:21.319><c> GPT</c><00:03:22.319><c> uh</c><00:03:22.480><c> so</c><00:03:22.799><c> if</c><00:03:22.920><c> you</c>\nrecent Trend since Chad GPT uh so if you\nrecent Trend since Chad GPT uh so if you ever<00:03:23.280><c> heard</c><00:03:23.440><c> of</c><00:03:23.560><c> gpt3</c><00:03:24.080><c> or</c><00:03:24.239><c> gpt2</c><00:03:25.080><c> that's</c><00:03:25.280><c> really</c>\never heard of gpt3 or gpt2 that's really\never heard of gpt3 or gpt2 that's really pre-training<00:03:26.280><c> land</c><00:03:27.280><c> uh</c><00:03:27.519><c> if</c><00:03:27.640><c> you</c><00:03:27.840><c> heard</c><00:03:28.040><c> of</c>\npre-training land uh if you heard of\npre-training land uh if you heard of chat<00:03:28.400><c> GPT</c><00:03:28.720><c> which</c><00:03:28.840><c> you</c><00:03:28.959><c> probably</c><00:03:29.319><c> have</c><00:03:29.840><c> this</c><00:03:29.959><c> is</c>\nchat GPT which you probably have this is\nchat GPT which you probably have this is really<00:03:30.360><c> posttraining</c><00:03:30.799><c> land</c><00:03:31.799><c> uh</c><00:03:31.879><c> so</c><00:03:32.040><c> I'll</c><00:03:32.159><c> be</c>\nreally posttraining land uh so I'll be\nreally posttraining land uh so I'll be talking<00:03:32.599><c> about</c><00:03:32.920><c> both</c><00:03:33.280><c> but</c><00:03:33.439><c> I'll</c><00:03:33.560><c> start</c><00:03:33.799><c> with</c>\ntalking about both but I'll start with\ntalking about both but I'll start with pre-training<00:03:35.200><c> and</c><00:03:35.439><c> uh</c><00:03:35.640><c> specifically</c><00:03:36.120><c> I'll</c>\npre-training and uh specifically I'll\npre-training and uh specifically I'll talk<00:03:36.879><c> about</c><00:03:37.200><c> what</c><00:03:37.360><c> is</c><00:03:37.480><c> the</c><00:03:37.879><c> task</c><00:03:38.360><c> of</c>\ntalk about what is the task of\ntalk about what is the task of pre-training<00:03:39.040><c> llms</c><00:03:39.840><c> and</c><00:03:40.000><c> what</c><00:03:40.080><c> is</c><00:03:40.200><c> the</c><00:03:40.319><c> laws</c>\npre-training llms and what is the laws\npre-training llms and what is the laws that<00:03:40.720><c> people</c><00:03:41.120><c> actually</c>\nthat people actually\nthat people actually use<00:03:43.280><c> so</c><00:03:43.519><c> language</c><00:03:43.879><c> modeling</c><00:03:44.480><c> this</c><00:03:44.560><c> is</c><00:03:44.720><c> a</c><00:03:45.360><c> quick</c>\nuse so language modeling this is a quick\nuse so language modeling this is a quick recap<00:03:46.959><c> uh</c><00:03:47.159><c> language</c><00:03:47.480><c> models</c><00:03:47.799><c> at</c><00:03:47.920><c> a</c><00:03:48.040><c> high</c><00:03:48.280><c> level</c>\nrecap uh language models at a high level\nrecap uh language models at a high level are<00:03:48.879><c> simply</c><00:03:49.319><c> models</c><00:03:49.840><c> of</c><00:03:50.080><c> probability</c>\nare simply models of probability\nare simply models of probability distribution<00:03:51.159><c> over</c><00:03:51.439><c> sequences</c><00:03:52.239><c> of</c><00:03:52.400><c> tokens</c><00:03:52.799><c> or</c>\ndistribution over sequences of tokens or\ndistribution over sequences of tokens or of<00:03:53.120><c> words</c><00:03:53.599><c> so</c><00:03:53.799><c> it's</c><00:03:54.040><c> basically</c><00:03:54.519><c> some</c><00:03:55.360><c> uh</c><00:03:55.560><c> model</c>\nof words so it's basically some uh model\nof words so it's basically some uh model of<00:03:56.239><c> P</c><00:03:56.480><c> of</c><00:03:56.720><c> X1</c><00:03:57.200><c> to</c><00:03:57.480><c> XL</c><00:03:58.000><c> where</c><00:03:58.200><c> X1</c><00:03:58.680><c> is</c><00:03:58.879><c> basically</c>\nof P of X1 to XL where X1 is basically\nof P of X1 to XL where X1 is basically word<00:03:59.560><c> one</c><00:03:59.879><c> and</c><00:04:00.040><c> Excel</c><00:04:00.360><c> is</c><00:04:00.519><c> the</c><00:04:00.680><c> last</c><00:04:00.959><c> one</c><00:04:01.560><c> in</c>\nword one and Excel is the last one in\nword one and Excel is the last one in the<00:04:01.840><c> sequence</c><00:04:02.280><c> or</c><00:04:02.400><c> in</c><00:04:02.519><c> the</c><00:04:03.079><c> sentence</c><00:04:04.079><c> um</c><00:04:04.280><c> so</c>\nthe sequence or in the sentence um so\nthe sequence or in the sentence um so very<00:04:04.680><c> concretely</c><00:04:05.319><c> if</c><00:04:05.400><c> you</c><00:04:05.519><c> have</c><00:04:05.640><c> a</c><00:04:05.799><c> sentence</c>\nvery concretely if you have a sentence\nvery concretely if you have a sentence like<00:04:06.480><c> the</c><00:04:06.760><c> mouse</c><00:04:07.120><c> ate</c><00:04:07.400><c> the</c><00:04:07.560><c> cheese</c><00:04:08.319><c> what</c><00:04:08.480><c> the</c>\nlike the mouse ate the cheese what the\nlike the mouse ate the cheese what the language<00:04:08.920><c> model</c><00:04:09.319><c> gives</c><00:04:09.560><c> you</c><00:04:09.959><c> is</c><00:04:10.120><c> simply</c><00:04:10.720><c> a</c>\nlanguage model gives you is simply a\nlanguage model gives you is simply a probability<00:04:11.959><c> of</c><00:04:12.360><c> this</c><00:04:12.760><c> sentence</c><00:04:13.200><c> being</c>\nprobability of this sentence being\nprobability of this sentence being uttered<00:04:13.840><c> by</c><00:04:13.959><c> a</c><00:04:14.079><c> human</c><00:04:14.360><c> or</c><00:04:14.560><c> being</c><00:04:14.760><c> found</c><00:04:15.120><c> on</c><00:04:15.439><c> on</c>\nuttered by a human or being found on on\nuttered by a human or being found on on online<00:04:17.079><c> uh</c><00:04:17.160><c> so</c><00:04:17.320><c> if</c><00:04:17.400><c> you</c><00:04:17.519><c> have</c><00:04:17.680><c> another</c>\nonline uh so if you have another\nonline uh so if you have another sentence<00:04:18.600><c> like</c><00:04:18.880><c> the</c><00:04:19.199><c> the</c><00:04:19.400><c> mouse</c><00:04:20.199><c> at</c><00:04:20.720><c> cheese</c><00:04:21.720><c> uh</c>\nsentence like the the mouse at cheese uh\nsentence like the the mouse at cheese uh here<00:04:22.199><c> there's</c><00:04:22.639><c> grammatical</c><00:04:23.160><c> mistakes</c><00:04:23.600><c> so</c><00:04:23.800><c> the</c>\nhere there's grammatical mistakes so the\nhere there's grammatical mistakes so the model<00:04:24.160><c> should</c><00:04:24.360><c> know</c><00:04:24.600><c> that</c><00:04:24.759><c> this</c><00:04:25.320><c> uh</c><00:04:25.520><c> should</c>\nmodel should know that this uh should\nmodel should know that this uh should have<00:04:25.880><c> some</c><00:04:26.120><c> syntactic</c><00:04:26.840><c> knowledge</c><00:04:27.400><c> so</c><00:04:27.520><c> it</c>\nhave some syntactic knowledge so it\nhave some syntactic knowledge so it should<00:04:27.800><c> know</c><00:04:27.960><c> that</c><00:04:28.199><c> this</c><00:04:28.520><c> has</c><00:04:28.800><c> less</c>\nshould know that this has less\nshould know that this has less likelihood<00:04:30.160><c> of</c><00:04:30.360><c> appearing</c>\nlikelihood of appearing\nlikelihood of appearing online<00:04:32.479><c> uh</c><00:04:32.639><c> if</c><00:04:32.720><c> you</c><00:04:32.880><c> have</c><00:04:33.080><c> another</c><00:04:33.720><c> sentence</c>\nonline uh if you have another sentence\nonline uh if you have another sentence like<00:04:34.440><c> the</c><00:04:34.639><c> cheese</c><00:04:35.080><c> ate</c><00:04:35.400><c> the</c><00:04:35.560><c> mouse</c><00:04:36.440><c> uh</c><00:04:36.560><c> then</c>\nlike the cheese ate the mouse uh then\nlike the cheese ate the mouse uh then the<00:04:36.880><c> model</c><00:04:37.199><c> should</c><00:04:37.400><c> hopefully</c><00:04:37.840><c> know</c><00:04:38.520><c> about</c>\nthe model should hopefully know about\nthe model should hopefully know about the<00:04:39.120><c> fact</c><00:04:39.440><c> that</c><00:04:39.800><c> usually</c><00:04:40.240><c> cheese</c><00:04:40.600><c> don't</c><00:04:40.800><c> eat</c>\nthe fact that usually cheese don't eat\nthe fact that usually cheese don't eat Mouse<00:04:41.880><c> um</c><00:04:42.120><c> so</c><00:04:42.360><c> there's</c><00:04:42.560><c> some</c><00:04:42.759><c> semantic</c>\nMouse um so there's some semantic\nMouse um so there's some semantic knowledge<00:04:43.520><c> and</c><00:04:43.680><c> this</c><00:04:43.759><c> is</c><00:04:43.960><c> less</c><00:04:44.160><c> likely</c><00:04:44.400><c> than</c>\nknowledge and this is less likely than\nknowledge and this is less likely than the<00:04:44.639><c> first</c><00:04:44.880><c> sentence</c><00:04:45.320><c> so</c><00:04:45.479><c> this</c><00:04:45.560><c> is</c><00:04:45.759><c> basically</c>\nthe first sentence so this is basically\nthe first sentence so this is basically at<00:04:46.160><c> a</c><00:04:46.240><c> high</c><00:04:46.400><c> level</c><00:04:46.639><c> what</c><00:04:46.840><c> language</c><00:04:47.199><c> models</c><00:04:47.960><c> are</c>\nat a high level what language models are\nat a high level what language models are um<00:04:50.240><c> one</c><00:04:50.440><c> word</c><00:04:50.680><c> that</c><00:04:50.759><c> you</c><00:04:50.960><c> probably</c><00:04:51.280><c> have</c><00:04:51.440><c> been</c>\num one word that you probably have been\num one word that you probably have been hearing<00:04:51.840><c> a</c><00:04:51.919><c> lot</c><00:04:52.039><c> in</c><00:04:52.160><c> the</c><00:04:52.240><c> news</c><00:04:52.479><c> are</c><00:04:52.639><c> generative</c>\nhearing a lot in the news are generative\nhearing a lot in the news are generative models<00:04:54.000><c> uh</c><00:04:54.120><c> so</c><00:04:54.320><c> this</c><00:04:54.440><c> is</c><00:04:54.639><c> just</c><00:04:54.800><c> something</c><00:04:55.120><c> that</c>\nmodels uh so this is just something that\nmodels uh so this is just something that can<00:04:55.520><c> generate</c><00:04:56.240><c> models</c><00:04:56.600><c> that</c><00:04:56.720><c> can</c><00:04:56.840><c> generate</c>\ncan generate models that can generate\ncan generate models that can generate sentences<00:04:57.800><c> or</c><00:04:57.960><c> can</c><00:04:58.240><c> generate</c><00:04:58.600><c> some</c><00:04:58.840><c> data</c><00:04:59.479><c> uh</c>\nsentences or can generate some data uh\nsentences or can generate some data uh the<00:04:59.800><c> reason</c><00:05:00.039><c> why</c><00:05:00.160><c> we</c><00:05:00.240><c> say</c><00:05:00.400><c> language</c><00:05:00.720><c> models</c>\nthe reason why we say language models\nthe reason why we say language models are<00:05:01.160><c> generative</c><00:05:01.479><c> models</c><00:05:01.840><c> is</c><00:05:01.919><c> that</c><00:05:02.120><c> once</c><00:05:02.280><c> you</c>\nare generative models is that once you\nare generative models is that once you have<00:05:03.000><c> a</c><00:05:03.160><c> model</c><00:05:03.440><c> of</c><00:05:03.560><c> a</c><00:05:03.680><c> distribution</c><00:05:04.320><c> you</c><00:05:04.440><c> can</c>\nhave a model of a distribution you can\nhave a model of a distribution you can simply<00:05:04.919><c> sample</c><00:05:05.320><c> from</c><00:05:05.520><c> this</c><00:05:05.680><c> model</c><00:05:06.160><c> and</c><00:05:06.280><c> now</c><00:05:06.400><c> we</c>\nsimply sample from this model and now we\nsimply sample from this model and now we can<00:05:06.600><c> generate</c><00:05:07.000><c> data</c><00:05:07.880><c> uh</c><00:05:07.960><c> so</c><00:05:08.080><c> you</c><00:05:08.199><c> can</c><00:05:08.320><c> generate</c>\ncan generate data uh so you can generate\ncan generate data uh so you can generate sentences<00:05:09.840><c> uh</c><00:05:10.039><c> using</c><00:05:10.440><c> a</c><00:05:10.600><c> language</c>\nsentences uh using a language\nsentences uh using a language model<00:05:12.560><c> so</c><00:05:12.759><c> the</c><00:05:12.919><c> type</c><00:05:13.080><c> of</c><00:05:13.240><c> models</c><00:05:13.680><c> that</c><00:05:14.039><c> uh</c>\nmodel so the type of models that uh\nmodel so the type of models that uh people<00:05:14.440><c> are</c><00:05:14.639><c> all</c><00:05:14.880><c> currently</c><00:05:15.280><c> using</c><00:05:15.680><c> are</c><00:05:15.880><c> what</c>\npeople are all currently using are what\npeople are all currently using are what we<00:05:16.160><c> call</c><00:05:16.440><c> Auto</c><00:05:16.800><c> regressive</c><00:05:17.759><c> language</c><00:05:18.199><c> models</c>\nwe call Auto regressive language models\nwe call Auto regressive language models and<00:05:19.319><c> the</c><00:05:19.600><c> key</c><00:05:20.080><c> idea</c><00:05:20.440><c> of</c><00:05:20.600><c> autor</c><00:05:20.880><c> regressive</c>\nand the key idea of autor regressive\nand the key idea of autor regressive language<00:05:21.560><c> models</c><00:05:22.080><c> is</c><00:05:22.240><c> that</c><00:05:22.400><c> you</c><00:05:22.560><c> take</c><00:05:23.199><c> this</c>\nlanguage models is that you take this\nlanguage models is that you take this distribution<00:05:24.360><c> over</c><00:05:24.759><c> words</c><00:05:25.639><c> and</c><00:05:25.800><c> you</c>\ndistribution over words and you\ndistribution over words and you basically<00:05:26.520><c> decompose</c><00:05:27.080><c> it</c><00:05:27.400><c> into</c><00:05:27.639><c> the</c><00:05:28.160><c> into</c><00:05:28.720><c> the</c>\nbasically decompose it into the into the\nbasically decompose it into the into the distribution<00:05:29.400><c> of</c><00:05:29.759><c> the</c><00:05:29.919><c> first</c><00:05:30.240><c> word</c><00:05:30.840><c> multiply</c>\ndistribution of the first word multiply\ndistribution of the first word multiply the<00:05:31.720><c> by</c><00:05:31.800><c> the</c><00:05:31.919><c> distribution</c><00:05:32.720><c> of</c><00:05:32.919><c> or</c><00:05:33.039><c> the</c>\nthe by the distribution of or the\nthe by the distribution of or the likelihood<00:05:33.560><c> of</c><00:05:33.639><c> the</c><00:05:33.759><c> distribution</c><00:05:34.360><c> of</c><00:05:34.600><c> the</c>\nlikelihood of the distribution of the\nlikelihood of the distribution of the second<00:05:35.080><c> word</c><00:05:35.479><c> given</c><00:05:35.759><c> the</c><00:05:35.919><c> first</c><00:05:36.199><c> word</c><00:05:36.919><c> uh</c>\nsecond word given the first word uh\nsecond word given the first word uh multiply<00:05:37.560><c> by</c><00:05:37.759><c> P</c><00:05:38.000><c> of</c><00:05:38.160><c> the</c><00:05:38.280><c> third</c><00:05:38.600><c> word</c><00:05:39.039><c> given</c>\nmultiply by P of the third word given\nmultiply by P of the third word given the<00:05:39.400><c> first</c><00:05:39.639><c> two</c><00:05:39.960><c> words</c><00:05:40.960><c> um</c><00:05:41.319><c> so</c><00:05:41.680><c> there's</c><00:05:41.840><c> no</c>\nthe first two words um so there's no\nthe first two words um so there's no approximation<00:05:42.680><c> here</c><00:05:42.919><c> this</c><00:05:43.000><c> is</c><00:05:43.160><c> just</c><00:05:43.280><c> the</c>\napproximation here this is just the\napproximation here this is just the chain<00:05:43.600><c> rule</c><00:05:43.800><c> of</c><00:05:43.919><c> probability</c><00:05:44.440><c> which</c><00:05:44.560><c> you</c>\nchain rule of probability which you\nchain rule of probability which you hopefully<00:05:45.080><c> all</c><00:05:45.240><c> know</c><00:05:45.440><c> about</c><00:05:46.080><c> uh</c><00:05:46.240><c> really</c><00:05:46.400><c> no</c>\nhopefully all know about uh really no\nhopefully all know about uh really no approximation<00:05:47.240><c> this</c><00:05:47.360><c> is</c><00:05:47.520><c> just</c><00:05:47.720><c> one</c><00:05:48.000><c> way</c><00:05:48.199><c> of</c>\napproximation this is just one way of\napproximation this is just one way of modeling<00:05:48.960><c> a</c>\nmodeling a\nmodeling a distribution<00:05:50.720><c> uh</c><00:05:50.840><c> so</c><00:05:51.080><c> slightly</c><00:05:51.400><c> more</c>\ndistribution uh so slightly more\ndistribution uh so slightly more concisely<00:05:52.039><c> you</c><00:05:52.120><c> can</c><00:05:52.319><c> write</c><00:05:52.440><c> it</c><00:05:52.560><c> as</c><00:05:52.680><c> a</c><00:05:52.880><c> product</c>\nconcisely you can write it as a product\nconcisely you can write it as a product of<00:05:53.840><c> U</c><00:05:54.360><c> of</c><00:05:54.600><c> PS</c><00:05:55.440><c> of</c><00:05:55.600><c> the</c><00:05:55.800><c> next</c><00:05:56.080><c> word</c><00:05:56.400><c> given</c>\nof U of PS of the next word given\nof U of PS of the next word given everything<00:05:57.160><c> which</c><00:05:57.360><c> happened</c><00:05:57.639><c> in</c><00:05:57.759><c> the</c><00:05:57.919><c> past</c><00:05:58.160><c> so</c>\neverything which happened in the past so\neverything which happened in the past so of<00:05:58.440><c> the</c><00:05:58.600><c> context</c><00:05:59.520><c> and</c><00:05:59.720><c> uh</c><00:05:59.800><c> so</c><00:06:00.080><c> this</c><00:06:00.319><c> this</c><00:06:00.440><c> is</c>\nof the context and uh so this this is\nof the context and uh so this this is what<00:06:00.680><c> we</c><00:06:00.840><c> call</c><00:06:01.080><c> Auto</c><00:06:01.360><c> regressive</c><00:06:01.759><c> language</c>\nwhat we call Auto regressive language\nwhat we call Auto regressive language models<00:06:02.840><c> again</c><00:06:03.080><c> this</c><00:06:03.240><c> is</c><00:06:03.759><c> really</c><00:06:04.080><c> not</c><00:06:04.280><c> the</c><00:06:04.440><c> only</c>\nmodels again this is really not the only\nmodels again this is really not the only way<00:06:04.960><c> of</c><00:06:05.319><c> modeling</c><00:06:05.720><c> distribution</c><00:06:06.280><c> this</c><00:06:06.400><c> is</c>\nway of modeling distribution this is\nway of modeling distribution this is just<00:06:06.800><c> one</c><00:06:07.080><c> way</c><00:06:07.880><c> uh</c><00:06:08.080><c> it</c><00:06:08.240><c> has</c><00:06:08.440><c> some</c><00:06:08.639><c> benefits</c><00:06:09.039><c> and</c>\njust one way uh it has some benefits and\njust one way uh it has some benefits and some<00:06:09.639><c> downsides</c><00:06:10.479><c> one</c><00:06:10.759><c> downside</c><00:06:11.280><c> of</c>\nsome downsides one downside of\nsome downsides one downside of autoaggressive<00:06:11.880><c> language</c><00:06:12.319><c> models</c><00:06:12.919><c> is</c><00:06:13.039><c> that</c>\nautoaggressive language models is that\nautoaggressive language models is that when<00:06:13.319><c> you</c><00:06:13.520><c> actually</c><00:06:13.759><c> sample</c><00:06:14.199><c> from</c><00:06:14.479><c> this</c>\nwhen you actually sample from this\nwhen you actually sample from this autoaggressive<00:06:15.400><c> language</c><00:06:15.800><c> model</c><00:06:16.039><c> you</c>\nautoaggressive language model you\nautoaggressive language model you basically<00:06:16.400><c> have</c><00:06:16.520><c> a</c><00:06:16.599><c> for</c><00:06:16.919><c> Loop</c><00:06:17.479><c> which</c>\nbasically have a for Loop which\nbasically have a for Loop which generates<00:06:18.240><c> the</c><00:06:18.400><c> next</c><00:06:18.680><c> word</c><00:06:19.240><c> then</c><00:06:19.680><c> conditions</c>\ngenerates the next word then conditions\ngenerates the next word then conditions on<00:06:20.520><c> that</c><00:06:20.720><c> next</c><00:06:20.960><c> word</c><00:06:21.400><c> and</c><00:06:21.479><c> then</c><00:06:21.639><c> regenerate</c><00:06:22.240><c> an</c>\non that next word and then regenerate an\non that next word and then regenerate an other<00:06:22.680><c> word</c><00:06:22.919><c> so</c><00:06:23.160><c> basically</c><00:06:23.800><c> if</c><00:06:23.880><c> you</c><00:06:24.000><c> have</c><00:06:24.120><c> a</c>\nother word so basically if you have a\nother word so basically if you have a longer<00:06:24.560><c> sentence</c><00:06:24.919><c> that</c><00:06:25.039><c> you</c><00:06:25.120><c> want</c><00:06:25.240><c> to</c>\nlonger sentence that you want to\nlonger sentence that you want to generate<00:06:26.080><c> you</c><00:06:26.319><c> it</c><00:06:26.479><c> takes</c><00:06:26.720><c> more</c><00:06:26.960><c> time</c><00:06:27.120><c> to</c>\ngenerate you it takes more time to\ngenerate you it takes more time to generate<00:06:27.639><c> it</c><00:06:28.240><c> uh</c><00:06:28.319><c> so</c><00:06:28.479><c> there</c><00:06:28.560><c> are</c><00:06:28.720><c> some</c>\ngenerate it uh so there are some\ngenerate it uh so there are some downsides<00:06:29.759><c> of</c><00:06:29.960><c> this</c><00:06:30.120><c> current</c><00:06:30.440><c> Paradigm</c><00:06:30.880><c> but</c>\ndownsides of this current Paradigm but\ndownsides of this current Paradigm but that's<00:06:31.280><c> what</c><00:06:31.520><c> we</c><00:06:32.080><c> currently</c><00:06:32.599><c> have</c><00:06:32.800><c> so</c><00:06:32.919><c> I'm</c>\nthat's what we currently have so I'm\nthat's what we currently have so I'm going<00:06:33.120><c> to</c><00:06:33.280><c> talk</c><00:06:33.440><c> about</c><00:06:33.680><c> this</c>\ngoing to talk about this\ngoing to talk about this one<00:06:35.880><c> uh</c><00:06:36.080><c> great</c><00:06:36.680><c> so</c><00:06:36.880><c> Auto</c><00:06:37.160><c> regressive</c><00:06:37.560><c> language</c>\none uh great so Auto regressive language\none uh great so Auto regressive language models<00:06:38.280><c> at</c><00:06:38.360><c> a</c><00:06:38.520><c> high</c><00:06:38.720><c> level</c><00:06:39.479><c> um</c><00:06:39.800><c> what</c><00:06:39.919><c> the</c><00:06:40.160><c> task</c>\nmodels at a high level um what the task\nmodels at a high level um what the task of<00:06:40.720><c> autoregressive</c><00:06:41.280><c> language</c><00:06:41.560><c> model</c><00:06:41.880><c> is</c><00:06:42.120><c> is</c>\nof autoregressive language model is is\nof autoregressive language model is is simply<00:06:42.479><c> predicting</c><00:06:42.840><c> the</c><00:06:43.000><c> next</c><00:06:43.199><c> word</c><00:06:43.440><c> as</c><00:06:43.520><c> I</c>\nsimply predicting the next word as I\nsimply predicting the next word as I just<00:06:43.800><c> said</c><00:06:44.199><c> so</c><00:06:44.319><c> if</c><00:06:44.400><c> you</c><00:06:44.520><c> have</c><00:06:44.599><c> a</c><00:06:44.720><c> sentence</c><00:06:45.039><c> like</c>\njust said so if you have a sentence like\njust said so if you have a sentence like she<00:06:45.479><c> likely</c><00:06:45.919><c> prefers</c><00:06:46.919><c> uh</c><00:06:47.160><c> one</c><00:06:47.440><c> potential</c><00:06:48.039><c> next</c>\nshe likely prefers uh one potential next\nshe likely prefers uh one potential next word<00:06:48.560><c> might</c><00:06:48.759><c> be</c><00:06:49.520><c> dogs</c><00:06:50.520><c> and</c><00:06:50.680><c> the</c><00:06:51.120><c> the</c><00:06:51.240><c> way</c><00:06:51.440><c> we</c><00:06:51.560><c> do</c>\nword might be dogs and the the way we do\nword might be dogs and the the way we do it<00:06:52.120><c> is</c><00:06:52.319><c> that</c><00:06:52.520><c> we</c><00:06:52.960><c> first</c><00:06:53.440><c> tokenize</c><00:06:54.440><c> so</c><00:06:54.639><c> you</c><00:06:54.800><c> take</c>\nit is that we first tokenize so you take\nit is that we first tokenize so you take these<00:06:55.240><c> words</c><00:06:55.800><c> or</c><00:06:56.080><c> subwords</c><00:06:56.680><c> you</c><00:06:56.840><c> tokenize</c>\nthese words or subwords you tokenize\nthese words or subwords you tokenize them<00:06:58.160><c> um</c><00:06:58.479><c> and</c><00:06:58.639><c> then</c><00:06:58.759><c> you</c><00:06:58.919><c> give</c><00:06:59.080><c> an</c><00:06:59.280><c> IDE</c><00:06:59.960><c> for</c>\nthem um and then you give an IDE for\nthem um and then you give an IDE for each<00:07:00.360><c> token</c><00:07:00.680><c> so</c><00:07:00.840><c> here</c><00:07:00.919><c> you</c><00:07:01.039><c> have</c><00:07:01.199><c> 1</c><00:07:01.360><c> 2</c><00:07:02.000><c> three</c><00:07:03.000><c> uh</c>\neach token so here you have 1 2 three uh\neach token so here you have 1 2 three uh then<00:07:03.360><c> you</c><00:07:03.720><c> pass</c><00:07:03.879><c> it</c><00:07:04.039><c> through</c><00:07:04.240><c> this</c><00:07:04.440><c> black</c><00:07:04.720><c> box</c>\nthen you pass it through this black box\nthen you pass it through this black box as<00:07:05.160><c> I</c><00:07:05.319><c> already</c><00:07:05.560><c> said</c><00:07:05.759><c> we're</c><00:07:05.919><c> not</c><00:07:06.000><c> going</c><00:07:06.120><c> to</c>\nas I already said we're not going to\nas I already said we're not going to talk<00:07:06.360><c> about</c><00:07:06.520><c> the</c><00:07:06.639><c> architecture</c><00:07:07.280><c> you</c><00:07:07.440><c> just</c>\ntalk about the architecture you just\ntalk about the architecture you just pass<00:07:07.879><c> it</c><00:07:08.560><c> pass</c><00:07:08.800><c> it</c><00:07:08.960><c> through</c><00:07:09.120><c> a</c><00:07:09.240><c> model</c><00:07:10.120><c> and</c><00:07:10.240><c> you</c>\npass it pass it through a model and you\npass it pass it through a model and you then<00:07:10.680><c> get</c><00:07:11.080><c> a</c><00:07:11.280><c> distribution</c><00:07:12.160><c> a</c><00:07:12.360><c> probability</c>\nthen get a distribution a probability\nthen get a distribution a probability distribution<00:07:13.720><c> over</c><00:07:14.000><c> the</c><00:07:14.199><c> next</c><00:07:14.520><c> word</c><00:07:14.879><c> over</c><00:07:15.080><c> the</c>\ndistribution over the next word over the\ndistribution over the next word over the next<00:07:15.879><c> token</c><00:07:16.879><c> and</c><00:07:17.080><c> then</c><00:07:17.520><c> you</c><00:07:17.800><c> sample</c><00:07:18.720><c> uh</c><00:07:18.840><c> from</c>\nnext token and then you sample uh from\nnext token and then you sample uh from this<00:07:19.240><c> distribution</c><00:07:20.039><c> you</c><00:07:20.199><c> get</c><00:07:20.319><c> a</c><00:07:20.479><c> new</c><00:07:20.720><c> token</c>\nthis distribution you get a new token\nthis distribution you get a new token and<00:07:21.599><c> then</c><00:07:21.720><c> you</c><00:07:21.879><c> DET</c><00:07:22.160><c> tokenize</c><00:07:22.840><c> so</c><00:07:22.960><c> you</c><00:07:23.080><c> get</c><00:07:23.160><c> a</c>\nand then you DET tokenize so you get a\nand then you DET tokenize so you get a new<00:07:23.440><c> ID</c><00:07:23.800><c> you</c><00:07:23.960><c> then</c><00:07:24.080><c> DET</c><00:07:24.319><c> toonize</c><00:07:24.720><c> and</c><00:07:25.240><c> that's</c>\nnew ID you then DET toonize and that's\nnew ID you then DET toonize and that's how<00:07:25.560><c> you</c><00:07:25.720><c> basically</c><00:07:26.080><c> sample</c><00:07:26.479><c> from</c><00:07:26.879><c> a</c><00:07:27.039><c> language</c>\nhow you basically sample from a language\nhow you basically sample from a language model<00:07:28.319><c> uh</c><00:07:28.440><c> one</c><00:07:28.599><c> thing</c><00:07:28.759><c> which</c><00:07:28.840><c> is</c><00:07:28.960><c> important</c><00:07:29.240><c> to</c>\nmodel uh one thing which is important to\nmodel uh one thing which is important to not<00:07:29.720><c> is</c><00:07:29.800><c> that</c><00:07:29.960><c> the</c><00:07:30.120><c> last</c><00:07:30.319><c> two</c><00:07:30.560><c> TS</c><00:07:31.000><c> uh</c><00:07:31.120><c> two</c><00:07:31.319><c> steps</c>\nnot is that the last two TS uh two steps\nnot is that the last two TS uh two steps are<00:07:31.759><c> actually</c><00:07:32.039><c> only</c><00:07:32.360><c> need</c><00:07:32.599><c> needed</c><00:07:32.960><c> during</c>\nare actually only need needed during\nare actually only need needed during inference<00:07:34.240><c> uh</c><00:07:34.360><c> when</c><00:07:34.479><c> you</c><00:07:34.560><c> do</c><00:07:34.759><c> training</c><00:07:35.400><c> you</c>\ninference uh when you do training you\ninference uh when you do training you just<00:07:35.720><c> need</c><00:07:35.879><c> to</c><00:07:36.080><c> predict</c><00:07:36.840><c> uh</c><00:07:36.919><c> the</c><00:07:37.039><c> most</c><00:07:37.240><c> likely</c>\njust need to predict uh the most likely\njust need to predict uh the most likely token<00:07:38.039><c> and</c><00:07:38.120><c> you</c><00:07:38.199><c> can</c><00:07:38.400><c> just</c><00:07:38.599><c> compare</c><00:07:39.080><c> to</c><00:07:39.240><c> the</c>\ntoken and you can just compare to the\ntoken and you can just compare to the real<00:07:39.680><c> token</c><00:07:40.199><c> which</c><00:07:40.400><c> happen</c><00:07:40.680><c> in</c><00:07:40.879><c> practice</c><00:07:41.479><c> and</c>\nreal token which happen in practice and\nreal token which happen in practice and then<00:07:41.759><c> you</c><00:07:42.400><c> basically</c><00:07:42.840><c> change</c><00:07:43.240><c> the</c><00:07:43.400><c> weights</c><00:07:43.800><c> of</c>\nthen you basically change the weights of\nthen you basically change the weights of your<00:07:44.080><c> model</c><00:07:44.639><c> to</c><00:07:44.840><c> increase</c><00:07:45.199><c> the</c><00:07:45.319><c> probability</c>\nyour model to increase the probability\nyour model to increase the probability of<00:07:45.840><c> generating</c><00:07:46.319><c> that</c>\nof generating that\nof generating that token<00:07:49.000><c> um</c><00:07:49.560><c> great</c><00:07:50.120><c> so</c><00:07:50.440><c> autoaggressive</c><00:07:50.960><c> neural</c>\ntoken um great so autoaggressive neural\ntoken um great so autoaggressive neural language<00:07:51.759><c> models</c><00:07:52.400><c> so</c><00:07:52.560><c> to</c><00:07:52.680><c> be</c><00:07:52.800><c> slightly</c><00:07:53.120><c> more</c>\nlanguage models so to be slightly more\nlanguage models so to be slightly more specific<00:07:53.720><c> still</c><00:07:53.960><c> without</c><00:07:54.280><c> talking</c><00:07:54.479><c> about</c><00:07:54.639><c> the</c>\nspecific still without talking about the\nspecific still without talking about the architecture<00:07:55.919><c> uh</c><00:07:56.039><c> the</c><00:07:56.159><c> first</c><00:07:56.360><c> thing</c><00:07:56.520><c> we</c><00:07:56.639><c> do</c><00:07:57.120><c> is</c>\narchitecture uh the first thing we do is\narchitecture uh the first thing we do is that<00:07:57.440><c> we</c><00:07:57.639><c> have</c><00:07:57.879><c> all</c><00:07:58.039><c> of</c><00:07:58.240><c> these</c><00:07:58.680><c> oh</c><00:07:58.840><c> sorry</c><00:07:59.159><c> yes</c>\nthat we have all of these oh sorry yes\nthat we have all of these oh sorry yes on<00:07:59.759><c> the</c><00:08:00.000><c> previous</c><00:08:00.400><c> slide</c><00:08:00.960><c> when</c><00:08:01.159><c> you're</c>\non the previous slide when you're\non the previous slide when you're predicting<00:08:01.960><c> the</c><00:08:02.120><c> probability</c><00:08:02.520><c> of</c><00:08:02.560><c> the</c><00:08:02.720><c> next</c>\npredicting the probability of the next\npredicting the probability of the next tokens<00:08:03.240><c> does</c><00:08:03.360><c> this</c><00:08:03.520><c> mean</c><00:08:03.680><c> that</c><00:08:03.800><c> your</c><00:08:04.000><c> final</c>\ntokens does this mean that your final\ntokens does this mean that your final like<00:08:04.879><c> output</c><00:08:05.319><c> VOR</c><00:08:05.759><c> has</c><00:08:05.879><c> to</c><00:08:06.000><c> be</c><00:08:06.280><c> the</c><00:08:06.360><c> same</c>\nlike output VOR has to be the same\nlike output VOR has to be the same dimensionality<00:08:07.680><c> as</c><00:08:07.840><c> the</c><00:08:08.000><c> number</c><00:08:08.240><c> of</c><00:08:08.400><c> tokens</c>\ndimensionality as the number of tokens\ndimensionality as the number of tokens that<00:08:08.840><c> you</c><00:08:09.039><c> have</c><00:08:09.440><c> yes</c><00:08:10.440><c> how</c><00:08:10.520><c> do</c><00:08:10.599><c> you</c><00:08:10.759><c> deal</c><00:08:11.000><c> with</c>\nthat you have yes how do you deal with\nthat you have yes how do you deal with like<00:08:11.400><c> if</c><00:08:11.520><c> you</c><00:08:11.960><c> have</c><00:08:12.280><c> more</c><00:08:12.560><c> to</c><00:08:12.879><c> like</c><00:08:13.000><c> if</c><00:08:13.080><c> you're</c>\nlike if you have more to like if you're\nlike if you have more to like if you're adding<00:08:13.800><c> more</c><00:08:14.000><c> tokens</c><00:08:14.280><c> to</c><00:08:14.400><c> your</c><00:08:14.520><c> cor</c><00:08:15.479><c> something</c>\nadding more tokens to your cor something\nadding more tokens to your cor something yeah<00:08:16.759><c> so</c><00:08:16.879><c> we're</c><00:08:17.000><c> going</c><00:08:17.080><c> to</c><00:08:17.199><c> talk</c><00:08:17.360><c> about</c>\nyeah so we're going to talk about\nyeah so we're going to talk about tokenization<00:08:18.599><c> actually</c><00:08:18.960><c> later</c><00:08:19.599><c> uh</c><00:08:19.720><c> so</c><00:08:19.840><c> you</c>\ntokenization actually later uh so you\ntokenization actually later uh so you will<00:08:20.120><c> get</c><00:08:20.360><c> some</c><00:08:20.599><c> sense</c><00:08:20.840><c> of</c><00:08:21.080><c> this</c><00:08:21.720><c> you</c>\nwill get some sense of this you\nwill get some sense of this you basically<00:08:22.520><c> can</c><00:08:22.840><c> deal</c><00:08:23.479><c> with</c><00:08:23.680><c> adding</c><00:08:24.039><c> new</c>\nbasically can deal with adding new\nbasically can deal with adding new tokens<00:08:25.000><c> I</c><00:08:25.159><c> am</c><00:08:25.360><c> I'm</c><00:08:25.560><c> kind</c><00:08:25.680><c> of</c><00:08:25.840><c> exaggerating</c>\ntokens I am I'm kind of exaggerating\ntokens I am I'm kind of exaggerating there<00:08:26.520><c> are</c><00:08:26.720><c> methods</c><00:08:27.000><c> for</c><00:08:27.159><c> doing</c><00:08:27.360><c> it</c><00:08:27.520><c> but</c>\nthere are methods for doing it but\nthere are methods for doing it but essentially<00:08:28.039><c> people</c><00:08:28.280><c> don't</c><00:08:28.479><c> do</c><00:08:28.639><c> it</c><00:08:29.319><c> um</c><00:08:29.879><c> so</c>\nessentially people don't do it um so\nessentially people don't do it um so it's<00:08:30.919><c> really</c><00:08:31.199><c> important</c><00:08:31.560><c> to</c><00:08:31.759><c> think</c><00:08:32.200><c> about</c><00:08:32.399><c> how</c>\nit's really important to think about how\nit's really important to think about how you<00:08:32.640><c> tokenize</c><00:08:33.120><c> your</c><00:08:33.240><c> text</c><00:08:33.479><c> and</c><00:08:33.560><c> that's</c><00:08:33.680><c> why</c>\nyou tokenize your text and that's why\nyou tokenize your text and that's why we'll<00:08:34.000><c> talk</c><00:08:34.200><c> about</c><00:08:34.399><c> that</c><00:08:34.599><c> later</c><00:08:35.560><c> but</c><00:08:35.680><c> it's</c><00:08:35.800><c> a</c>\nwe'll talk about that later but it's a\nwe'll talk about that later but it's a very<00:08:36.159><c> good</c><00:08:36.320><c> point</c><00:08:36.479><c> to</c><00:08:36.599><c> notice</c><00:08:37.000><c> that</c><00:08:37.120><c> you</c>\nvery good point to notice that you\nvery good point to notice that you basically<00:08:37.640><c> the</c><00:08:37.800><c> vocabulary</c><00:08:38.320><c> size</c><00:08:38.519><c> so</c><00:08:38.680><c> the</c>\nbasically the vocabulary size so the\nbasically the vocabulary size so the number<00:08:38.959><c> of</c><00:08:39.080><c> tokens</c><00:08:39.399><c> that</c><00:08:39.519><c> you</c><00:08:39.680><c> have</c><00:08:40.039><c> is</c>\nnumber of tokens that you have is\nnumber of tokens that you have is essentially<00:08:40.599><c> the</c><00:08:40.719><c> output</c><00:08:41.560><c> of</c><00:08:41.760><c> your</c><00:08:42.200><c> uh</c>\nessentially the output of your uh\nessentially the output of your uh language<00:08:42.719><c> model</c><00:08:43.200><c> so</c><00:08:43.360><c> it's</c><00:08:43.560><c> actually</c><00:08:43.800><c> pretty</c>\nlanguage model so it's actually pretty\nlanguage model so it's actually pretty pretty\npretty\npretty large<00:08:46.200><c> okay</c><00:08:46.320><c> so</c><00:08:46.480><c> autoaggressive</c><00:08:47.000><c> new</c>\nlarge okay so autoaggressive new\nlarge okay so autoaggressive new language<00:08:47.800><c> models</c><00:08:48.800><c> first</c><00:08:49.040><c> thing</c><00:08:49.160><c> you</c><00:08:49.320><c> do</c><00:08:49.600><c> is</c>\nlanguage models first thing you do is\nlanguage models first thing you do is that<00:08:49.880><c> you</c><00:08:50.040><c> take</c><00:08:50.360><c> every</c><00:08:50.600><c> word</c><00:08:50.800><c> or</c><00:08:50.959><c> every</c><00:08:51.200><c> token</c>\nthat you take every word or every token\nthat you take every word or every token you<00:08:52.360><c> embed</c><00:08:52.800><c> them</c><00:08:53.000><c> so</c><00:08:53.160><c> you</c><00:08:53.279><c> get</c><00:08:53.560><c> a</c><00:08:54.000><c> um</c><00:08:54.800><c> some</c>\nyou embed them so you get a um some\nyou embed them so you get a um some Vector<00:08:55.519><c> representation</c><00:08:56.120><c> for</c><00:08:56.320><c> each</c><00:08:56.480><c> of</c><00:08:56.640><c> these</c>\nVector representation for each of these\nVector representation for each of these tokens<00:08:58.040><c> um</c><00:08:58.360><c> you</c><00:08:58.560><c> pass</c><00:08:58.800><c> them</c><00:08:58.959><c> through</c><00:08:59.160><c> some</c><00:08:59.440><c> ual</c>\ntokens um you pass them through some ual\ntokens um you pass them through some ual Network<00:08:59.959><c> as</c><00:09:00.040><c> we</c><00:09:00.160><c> said</c><00:09:00.320><c> it's</c><00:09:00.440><c> a</c><00:09:00.560><c> Transformer</c>\nNetwork as we said it's a Transformer\nNetwork as we said it's a Transformer then<00:09:01.640><c> you</c><00:09:01.760><c> get</c><00:09:01.920><c> a</c><00:09:02.240><c> representation</c><00:09:03.240><c> for</c><00:09:03.720><c> all</c>\nthen you get a representation for all\nthen you get a representation for all the<00:09:04.160><c> word</c><00:09:04.800><c> in</c><00:09:05.079><c> all</c><00:09:05.240><c> the</c><00:09:05.399><c> words</c><00:09:05.760><c> in</c><00:09:05.880><c> the</c><00:09:06.079><c> context</c>\nthe word in all the words in the context\nthe word in all the words in the context so<00:09:06.800><c> it's</c><00:09:06.959><c> basically</c><00:09:07.279><c> representation</c><00:09:08.000><c> of</c><00:09:08.120><c> the</c>\nso it's basically representation of the\nso it's basically representation of the entire<00:09:08.839><c> sentence</c><00:09:09.839><c> uh</c><00:09:10.000><c> you</c><00:09:10.200><c> pass</c><00:09:10.360><c> it</c><00:09:10.519><c> through</c><00:09:10.680><c> a</c>\nentire sentence uh you pass it through a\nentire sentence uh you pass it through a linear<00:09:11.120><c> layer</c><00:09:11.800><c> as</c><00:09:11.920><c> you</c><00:09:12.120><c> just</c><00:09:12.279><c> said</c><00:09:12.720><c> to</c>\nlinear layer as you just said to\nlinear layer as you just said to basically<00:09:13.839><c> map</c><00:09:14.120><c> it</c><00:09:14.279><c> to</c><00:09:14.519><c> the</c><00:09:15.000><c> number</c><00:09:15.680><c> so</c><00:09:15.880><c> that</c>\nbasically map it to the number so that\nbasically map it to the number so that the<00:09:16.200><c> output</c><00:09:16.560><c> the</c><00:09:16.640><c> number</c><00:09:17.000><c> of</c><00:09:17.240><c> outputs</c><00:09:17.640><c> is</c><00:09:17.720><c> the</c>\nthe output the number of outputs is the\nthe output the number of outputs is the number<00:09:18.079><c> of</c><00:09:18.360><c> tokens</c><00:09:19.360><c> uh</c><00:09:19.640><c> you</c><00:09:19.839><c> then</c><00:09:20.040><c> pass</c><00:09:20.200><c> it</c>\nnumber of tokens uh you then pass it\nnumber of tokens uh you then pass it through<00:09:20.519><c> some</c><00:09:20.720><c> soft</c><00:09:21.040><c> Max</c><00:09:21.640><c> and</c><00:09:21.760><c> you</c><00:09:21.920><c> basically</c>\nthrough some soft Max and you basically\nthrough some soft Max and you basically get<00:09:23.000><c> uh</c><00:09:23.200><c> probity</c><00:09:23.800><c> distribution</c><00:09:24.800><c> over</c><00:09:25.440><c> the</c>\nget uh probity distribution over the\nget uh probity distribution over the next<00:09:25.959><c> words</c><00:09:26.519><c> given</c><00:09:27.120><c> every</c><00:09:27.440><c> word</c><00:09:27.640><c> in</c><00:09:27.720><c> the</c>\nnext words given every word in the\nnext words given every word in the context\ncontext\ncontext and<00:09:30.680><c> the</c><00:09:30.760><c> law</c><00:09:31.079><c> that</c><00:09:31.200><c> you</c><00:09:31.360><c> use</c><00:09:31.880><c> is</c><00:09:32.079><c> basically</c>\nand the law that you use is basically\nand the law that you use is basically it's<00:09:33.040><c> essentially</c><00:09:33.399><c> a</c><00:09:33.560><c> task</c><00:09:33.800><c> of</c><00:09:34.000><c> classifying</c>\nit's essentially a task of classifying\nit's essentially a task of classifying the<00:09:34.720><c> next</c><00:09:35.000><c> token</c><00:09:35.279><c> so</c><00:09:35.440><c> it's</c><00:09:35.519><c> a</c><00:09:35.680><c> very</c><00:09:35.880><c> simple</c>\nthe next token so it's a very simple\nthe next token so it's a very simple kind<00:09:36.399><c> of</c><00:09:36.519><c> machine</c><00:09:36.800><c> learning</c><00:09:37.120><c> task</c><00:09:37.560><c> so</c><00:09:37.680><c> you</c><00:09:37.800><c> use</c>\nkind of machine learning task so you use\nkind of machine learning task so you use the<00:09:38.079><c> cross</c><00:09:38.279><c> entry</c><00:09:38.600><c> P</c><00:09:38.800><c> loss</c><00:09:39.399><c> where</c><00:09:39.560><c> you</c>\nthe cross entry P loss where you\nthe cross entry P loss where you basically<00:09:40.360><c> you</c><00:09:40.720><c> look</c><00:09:41.120><c> at</c><00:09:41.640><c> the</c><00:09:42.120><c> actual</c><00:09:43.040><c> Target</c>\nbasically you look at the actual Target\nbasically you look at the actual Target that<00:09:44.160><c> happened</c><00:09:44.480><c> which</c><00:09:44.600><c> is</c><00:09:44.680><c> a</c><00:09:44.800><c> target</c>\nthat happened which is a target\nthat happened which is a target distribution<00:09:45.640><c> which</c><00:09:45.760><c> is</c><00:09:45.839><c> a</c><00:09:46.000><c> one</c><00:09:46.240><c> hot</c><00:09:46.440><c> encoding</c>\ndistribution which is a one hot encoding\ndistribution which is a one hot encoding which<00:09:47.120><c> here</c><00:09:47.279><c> in</c><00:09:47.440><c> this</c><00:09:47.760><c> in</c><00:09:47.920><c> this</c><00:09:48.200><c> case</c><00:09:48.480><c> says</c><00:09:49.040><c> I</c>\nwhich here in this in this case says I\nwhich here in this in this case says I saw<00:09:49.560><c> uh</c><00:09:50.040><c> the</c><00:09:50.200><c> real</c><00:09:50.440><c> word</c><00:09:50.680><c> that</c><00:09:50.839><c> happened</c><00:09:51.240><c> is</c>\nsaw uh the real word that happened is\nsaw uh the real word that happened is cat<00:09:51.880><c> so</c><00:09:52.000><c> that's</c><00:09:52.120><c> a</c><00:09:52.240><c> one</c><00:09:52.480><c> hot</c><00:09:53.200><c> um</c><00:09:53.920><c> distribution</c>\ncat so that's a one hot um distribution\ncat so that's a one hot um distribution over<00:09:55.279><c> cat</c><00:09:55.680><c> and</c><00:09:55.880><c> here</c><00:09:56.079><c> this</c><00:09:56.200><c> is</c><00:09:56.360><c> the</c><00:09:56.600><c> actual</c><00:09:57.560><c> uh</c>\nover cat and here this is the actual uh\nover cat and here this is the actual uh do<00:09:57.800><c> you</c><00:09:57.920><c> see</c><00:09:58.079><c> my</c><00:09:58.200><c> mouse</c><00:09:58.480><c> oh</c><00:09:58.640><c> yeah</c><00:09:58.839><c> this</c><00:09:58.959><c> is</c><00:09:59.040><c> the</c>\ndo you see my mouse oh yeah this is the\ndo you see my mouse oh yeah this is the distribtion<00:09:59.680><c> that</c><00:09:59.760><c> you</c><00:09:59.880><c> generated</c><00:10:00.600><c> and</c>\ndistribtion that you generated and\ndistribtion that you generated and basically<00:10:01.000><c> you</c><00:10:01.120><c> do</c><00:10:01.279><c> cross</c><00:10:01.519><c> entropy</c><00:10:01.959><c> which</c>\nbasically you do cross entropy which\nbasically you do cross entropy which really<00:10:02.480><c> just</c><00:10:02.760><c> increases</c><00:10:03.240><c> the</c><00:10:03.360><c> probability</c><00:10:03.720><c> of</c>\nreally just increases the probability of\nreally just increases the probability of generating<00:10:04.240><c> cat</c><00:10:04.440><c> and</c><00:10:04.600><c> decreases</c><00:10:05.360><c> all</c><00:10:05.560><c> the</c><00:10:05.880><c> the</c>\ngenerating cat and decreases all the the\ngenerating cat and decreases all the the probility<00:10:06.279><c> of</c><00:10:06.360><c> generating</c><00:10:06.800><c> all</c><00:10:06.959><c> the</c><00:10:07.040><c> other</c>\nprobility of generating all the other\nprobility of generating all the other tokens<00:10:08.200><c> one</c><00:10:08.399><c> thing</c><00:10:08.560><c> to</c><00:10:08.720><c> notice</c><00:10:09.519><c> is</c><00:10:09.720><c> that</c><00:10:09.959><c> as</c>\ntokens one thing to notice is that as\ntokens one thing to notice is that as you<00:10:10.200><c> all</c><00:10:10.399><c> know</c><00:10:10.839><c> again</c><00:10:11.560><c> uh</c><00:10:11.760><c> this</c><00:10:12.079><c> is</c><00:10:12.519><c> just</c>\nyou all know again uh this is just\nyou all know again uh this is just equivalent<00:10:13.360><c> to</c><00:10:13.600><c> maximizing</c><00:10:14.240><c> the</c><00:10:14.399><c> text</c><00:10:14.600><c> log</c>\nequivalent to maximizing the text log\nequivalent to maximizing the text log like<00:10:15.279><c> the</c><00:10:15.440><c> text</c><00:10:15.680><c> log</c><00:10:15.920><c> likelihood</c><00:10:16.640><c> because</c><00:10:16.760><c> you</c>\nlike the text log likelihood because you\nlike the text log likelihood because you can<00:10:17.040><c> just</c><00:10:17.480><c> rewrite</c><00:10:18.160><c> the</c><00:10:18.720><c> the</c><00:10:19.320><c> max</c><00:10:19.800><c> over</c><00:10:20.240><c> the</c>\ncan just rewrite the the max over the\ncan just rewrite the the max over the probability<00:10:21.480><c> of</c><00:10:21.920><c> um</c><00:10:22.120><c> this</c><00:10:22.279><c> autoregressive</c>\nprobability of um this autoregressive\nprobability of um this autoregressive language<00:10:23.160><c> moding</c><00:10:23.600><c> task</c><00:10:24.360><c> as</c><00:10:24.560><c> just</c><00:10:24.720><c> being</c><00:10:25.000><c> this</c>\nlanguage moding task as just being this\nlanguage moding task as just being this minimum<00:10:26.079><c> over</c><00:10:26.600><c> I</c><00:10:26.720><c> just</c><00:10:26.880><c> added</c><00:10:27.160><c> the</c><00:10:27.320><c> log</c><00:10:27.680><c> here</c>\nminimum over I just added the log here\nminimum over I just added the log here and<00:10:28.120><c> minus</c><00:10:28.880><c> which</c><00:10:29.000><c> is</c><00:10:29.399><c> just</c><00:10:29.519><c> the</c><00:10:29.600><c> minimum</c><00:10:29.959><c> of</c>\nand minus which is just the minimum of\nand minus which is just the minimum of the<00:10:30.200><c> loss</c><00:10:30.480><c> which</c><00:10:30.560><c> is</c><00:10:30.640><c> the</c><00:10:30.760><c> cross</c><00:10:30.959><c> enty</c><00:10:31.399><c> loss</c><00:10:31.600><c> so</c>\nthe loss which is the cross enty loss so\nthe loss which is the cross enty loss so basically<00:10:32.079><c> minimizing</c><00:10:32.560><c> the</c><00:10:32.680><c> loss</c><00:10:33.120><c> is</c><00:10:33.240><c> the</c>\nbasically minimizing the loss is the\nbasically minimizing the loss is the same<00:10:33.560><c> thing</c><00:10:33.760><c> as</c><00:10:33.959><c> maximizing</c><00:10:34.519><c> the</c><00:10:34.640><c> likelihood</c>\nsame thing as maximizing the likelihood\nsame thing as maximizing the likelihood of<00:10:35.639><c> your</c><00:10:35.920><c> text</c><00:10:36.920><c> any</c><00:10:37.120><c> question</c>\nquestions\nquestions\nquestions okay\nokay\nokay tokenizer<00:10:46.839><c> um</c><00:10:47.200><c> so</c><00:10:47.959><c> this</c><00:10:48.079><c> is</c><00:10:48.279><c> one</c><00:10:48.519><c> thing</c><00:10:48.720><c> that</c>\ntokenizer um so this is one thing that\ntokenizer um so this is one thing that people<00:10:49.120><c> usually</c><00:10:49.399><c> don't</c><00:10:49.680><c> talk</c><00:10:50.079><c> that</c><00:10:50.240><c> much</c>\npeople usually don't talk that much\npeople usually don't talk that much about<00:10:50.880><c> tokenizers</c><00:10:51.680><c> are</c><00:10:52.040><c> extremely</c><00:10:52.680><c> important</c>\nabout tokenizers are extremely important\nabout tokenizers are extremely important uh<00:10:53.519><c> so</c><00:10:53.639><c> it's</c><00:10:53.760><c> really</c><00:10:53.959><c> important</c><00:10:54.279><c> that</c><00:10:54.399><c> you</c>\nuh so it's really important that you\nuh so it's really important that you kind<00:10:54.680><c> of</c><00:10:55.079><c> understand</c><00:10:55.240><c> at</c><00:10:55.360><c> least</c><00:10:56.200><c> uh</c><00:10:56.399><c> what</c><00:10:56.519><c> they</c>\nkind of understand at least uh what they\nkind of understand at least uh what they do<00:10:56.800><c> at</c><00:10:56.920><c> a</c><00:10:57.040><c> high</c><00:10:57.240><c> level</c><00:10:58.040><c> so</c><00:10:58.279><c> why</c><00:10:58.440><c> do</c><00:10:58.519><c> we</c><00:10:58.639><c> need</c>\ndo at a high level so why do we need\ndo at a high level so why do we need token<00:10:59.560><c> in</c><00:10:59.639><c> the</c><00:10:59.760><c> first</c><00:11:00.040><c> place</c><00:11:01.040><c> uh</c><00:11:01.279><c> first</c><00:11:01.680><c> it's</c>\ntoken in the first place uh first it's\ntoken in the first place uh first it's more<00:11:02.079><c> General</c><00:11:02.440><c> than</c><00:11:02.639><c> words</c><00:11:03.079><c> so</c><00:11:03.360><c> one</c><00:11:03.639><c> simple</c>\nmore General than words so one simple\nmore General than words so one simple thing<00:11:04.240><c> that</c><00:11:04.320><c> you</c><00:11:04.480><c> might</c><00:11:04.639><c> think</c><00:11:04.880><c> is</c><00:11:05.120><c> oh</c><00:11:05.279><c> we're</c>\nthing that you might think is oh we're\nthing that you might think is oh we're just<00:11:05.600><c> going</c><00:11:05.720><c> to</c><00:11:05.839><c> take</c><00:11:06.079><c> every</c><00:11:06.279><c> word</c><00:11:06.519><c> that</c><00:11:06.639><c> we</c>\njust going to take every word that we\njust going to take every word that we will<00:11:07.000><c> have</c><00:11:07.440><c> you</c><00:11:07.680><c> just</c><00:11:07.839><c> say</c><00:11:08.279><c> every</c><00:11:08.560><c> word</c><00:11:08.880><c> is</c><00:11:09.000><c> a</c>\nwill have you just say every word is a\nwill have you just say every word is a new<00:11:09.440><c> is</c><00:11:09.519><c> a</c><00:11:09.639><c> token</c><00:11:09.920><c> in</c><00:11:10.040><c> its</c><00:11:10.160><c> own</c><00:11:11.040><c> um</c><00:11:11.399><c> but</c><00:11:11.600><c> then</c>\nnew is a token in its own um but then\nnew is a token in its own um but then what<00:11:12.120><c> happens</c><00:11:12.440><c> is</c><00:11:12.600><c> if</c><00:11:12.720><c> there's</c><00:11:12.880><c> a</c><00:11:13.040><c> typo</c><00:11:13.560><c> in</c>\nwhat happens is if there's a typo in\nwhat happens is if there's a typo in your<00:11:13.920><c> word</c><00:11:14.920><c> then</c><00:11:15.040><c> you</c><00:11:15.279><c> might</c><00:11:15.519><c> not</c><00:11:15.760><c> have</c><00:11:16.000><c> any</c>\nyour word then you might not have any\nyour word then you might not have any token<00:11:16.760><c> associated</c><00:11:17.760><c> with</c><00:11:18.160><c> this</c><00:11:18.600><c> this</c><00:11:18.839><c> word</c>\ntoken associated with this this word\ntoken associated with this this word with<00:11:19.320><c> a</c><00:11:19.440><c> typo</c><00:11:20.040><c> and</c><00:11:20.160><c> then</c><00:11:20.279><c> you</c><00:11:20.399><c> don't</c><00:11:20.639><c> know</c><00:11:20.839><c> how</c>\nwith a typo and then you don't know how\nwith a typo and then you don't know how to<00:11:21.200><c> actually</c><00:11:21.560><c> pass</c><00:11:21.959><c> this</c><00:11:22.079><c> word</c><00:11:22.279><c> with</c><00:11:22.480><c> a</c><00:11:22.600><c> typo</c>\nto actually pass this word with a typo\nto actually pass this word with a typo into<00:11:23.160><c> a</c><00:11:23.279><c> large</c><00:11:23.560><c> language</c><00:11:23.880><c> model</c><00:11:24.600><c> so</c><00:11:24.760><c> what</c><00:11:24.880><c> do</c>\ninto a large language model so what do\ninto a large language model so what do you<00:11:25.079><c> do</c><00:11:25.320><c> next</c><00:11:25.800><c> and</c><00:11:26.000><c> also</c><00:11:26.560><c> even</c><00:11:26.760><c> if</c><00:11:26.880><c> you</c><00:11:27.000><c> think</c>\nyou do next and also even if you think\nyou do next and also even if you think about<00:11:27.399><c> words</c><00:11:27.920><c> words</c><00:11:28.160><c> is</c><00:11:28.240><c> a</c><00:11:28.480><c> very</c><00:11:28.800><c> like</c><00:11:29.440><c> words</c>\nabout words words is a very like words\nabout words words is a very like words are<00:11:30.040><c> fine</c><00:11:30.279><c> with</c><00:11:30.480><c> like</c><00:11:30.600><c> Latin</c><00:11:30.959><c> based</c><00:11:31.360><c> languages</c>\nare fine with like Latin based languages\nare fine with like Latin based languages uh<00:11:32.519><c> but</c><00:11:32.680><c> if</c><00:11:32.800><c> you</c><00:11:32.959><c> think</c><00:11:33.200><c> about</c><00:11:33.519><c> a</c><00:11:33.680><c> language</c>\nuh but if you think about a language\nuh but if you think about a language like<00:11:34.200><c> taii</c><00:11:34.839><c> you</c><00:11:35.000><c> won't</c><00:11:35.320><c> have</c><00:11:35.480><c> a</c><00:11:35.600><c> simple</c><00:11:35.920><c> way</c><00:11:36.079><c> of</c>\nlike taii you won't have a simple way of\nlike taii you won't have a simple way of tokenizing<00:11:36.800><c> by</c><00:11:37.000><c> spaces</c><00:11:37.440><c> because</c><00:11:37.519><c> there</c><00:11:37.600><c> are</c>\ntokenizing by spaces because there are\ntokenizing by spaces because there are no<00:11:37.880><c> spaces</c><00:11:38.279><c> between</c><00:11:38.600><c> words</c><00:11:39.480><c> um</c><00:11:39.760><c> so</c><00:11:40.040><c> really</c><00:11:40.880><c> uh</c>\nno spaces between words um so really uh\nno spaces between words um so really uh tokens<00:11:41.360><c> are</c><00:11:41.519><c> much</c><00:11:41.680><c> more</c><00:11:41.839><c> General</c><00:11:42.399><c> Than</c><00:11:42.800><c> Words</c>\ntokens are much more General Than Words\ntokens are much more General Than Words first<00:11:44.000><c> thing</c><00:11:44.240><c> second</c><00:11:44.480><c> thing</c><00:11:44.600><c> that</c><00:11:44.720><c> you</c><00:11:44.839><c> might</c>\nfirst thing second thing that you might\nfirst thing second thing that you might think<00:11:45.480><c> is</c><00:11:45.639><c> that</c><00:11:45.800><c> you</c><00:11:46.040><c> might</c><00:11:46.279><c> tokenize</c><00:11:47.279><c> every</c>\nthink is that you might tokenize every\nthink is that you might tokenize every sentence<00:11:48.240><c> character</c><00:11:48.639><c> by</c><00:11:48.880><c> character</c><00:11:49.399><c> you</c>\nsentence character by character you\nsentence character by character you might<00:11:49.720><c> say</c><00:11:49.959><c> a</c><00:11:50.240><c> is</c><00:11:50.399><c> one</c><00:11:50.600><c> token</c><00:11:50.959><c> b</c><00:11:51.200><c> is</c><00:11:51.360><c> another</c>\nmight say a is one token b is another\nmight say a is one token b is another token<00:11:52.639><c> uh</c><00:11:52.880><c> that</c><00:11:53.040><c> would</c><00:11:53.320><c> actually</c><00:11:53.600><c> work</c><00:11:54.040><c> and</c>\ntoken uh that would actually work and\ntoken uh that would actually work and probably<00:11:54.560><c> very</c><00:11:54.800><c> well</c><00:11:55.440><c> the</c><00:11:55.639><c> issue</c><00:11:55.920><c> is</c><00:11:56.040><c> that</c>\nprobably very well the issue is that\nprobably very well the issue is that then<00:11:56.360><c> your</c><00:11:56.519><c> sequence</c><00:11:56.959><c> becomes</c><00:11:57.440><c> super</c><00:11:57.760><c> long</c>\nthen your sequence becomes super long\nthen your sequence becomes super long and<00:11:58.600><c> as</c><00:11:58.720><c> you</c><00:11:59.240><c> probably</c><00:11:59.519><c> remember</c><00:11:59.920><c> from</c><00:12:00.079><c> the</c>\nand as you probably remember from the\nand as you probably remember from the lecture<00:12:00.600><c> on</c><00:12:00.800><c> on</c><00:12:01.200><c> Transformers</c><00:12:02.200><c> uh</c><00:12:02.360><c> the</c>\nlecture on on Transformers uh the\nlecture on on Transformers uh the complexity<00:12:03.880><c> uh</c><00:12:04.079><c> grows</c><00:12:04.519><c> quadratically</c><00:12:05.440><c> with</c>\ncomplexity uh grows quadratically with\ncomplexity uh grows quadratically with the<00:12:05.720><c> length</c><00:12:06.000><c> of</c><00:12:06.120><c> sequences</c><00:12:06.839><c> so</c><00:12:07.000><c> you</c><00:12:07.320><c> really</c>\nthe length of sequences so you really\nthe length of sequences so you really don't<00:12:07.680><c> want</c><00:12:07.760><c> to</c><00:12:07.959><c> have</c><00:12:08.160><c> a</c><00:12:08.279><c> super</c><00:12:08.600><c> long</c><00:12:08.959><c> sequence</c>\ndon't want to have a super long sequence\ndon't want to have a super long sequence um<00:12:10.120><c> so</c><00:12:10.440><c> tokenizers</c><00:12:11.440><c> basically</c><00:12:11.959><c> try</c><00:12:12.639><c> to</c><00:12:13.040><c> deal</c>\num so tokenizers basically try to deal\num so tokenizers basically try to deal with<00:12:13.519><c> those</c><00:12:13.720><c> two</c><00:12:14.000><c> problems</c><00:12:14.839><c> and</c><00:12:15.399><c> give</c><00:12:15.800><c> common</c>\nwith those two problems and give common\nwith those two problems and give common subsequences<00:12:18.160><c> a</c><00:12:18.360><c> certain</c><00:12:18.720><c> token</c><00:12:19.480><c> and</c><00:12:19.760><c> usually</c>\nsubsequences a certain token and usually\nsubsequences a certain token and usually how<00:12:20.279><c> you</c><00:12:20.360><c> should</c><00:12:20.600><c> be</c><00:12:20.760><c> think</c><00:12:20.959><c> about</c><00:12:21.199><c> is</c><00:12:21.600><c> around</c>\nhow you should be think about is around\nhow you should be think about is around uh<00:12:22.600><c> an</c><00:12:22.800><c> average</c><00:12:23.639><c> every</c><00:12:23.880><c> token</c><00:12:24.120><c> is</c><00:12:24.240><c> around</c>\nuh an average every token is around\nuh an average every token is around three<00:12:24.720><c> four</c><00:12:25.000><c> letters</c>\nthree four letters\nthree four letters um<00:12:27.880><c> and</c><00:12:28.399><c> there</c><00:12:28.519><c> are</c><00:12:28.639><c> many</c><00:12:28.800><c> algorithm</c><00:12:29.399><c> for</c>\num and there are many algorithm for\num and there are many algorithm for tokenization<00:12:30.160><c> I'll</c><00:12:30.320><c> just</c><00:12:30.440><c> talk</c><00:12:30.600><c> about</c><00:12:30.800><c> one</c><00:12:30.920><c> of</c>\ntokenization I'll just talk about one of\ntokenization I'll just talk about one of them<00:12:31.199><c> to</c><00:12:31.320><c> give</c><00:12:31.440><c> you</c><00:12:31.519><c> a</c><00:12:31.639><c> high</c><00:12:31.839><c> level</c><00:12:32.639><c> which</c><00:12:32.760><c> is</c>\nthem to give you a high level which is\nthem to give you a high level which is what<00:12:33.000><c> we</c><00:12:33.120><c> call</c><00:12:33.320><c> bite</c><00:12:33.600><c> P</c><00:12:33.760><c> en</c><00:12:33.880><c> coding</c><00:12:34.160><c> which</c><00:12:34.240><c> is</c>\nwhat we call bite P en coding which is\nwhat we call bite P en coding which is actually<00:12:34.680><c> pretty</c><00:12:34.959><c> common</c><00:12:35.360><c> one</c><00:12:35.480><c> of</c><00:12:35.600><c> the</c><00:12:35.720><c> two</c>\nactually pretty common one of the two\nactually pretty common one of the two most<00:12:36.120><c> common</c><00:12:36.880><c> tokenizers</c><00:12:37.880><c> and</c><00:12:37.959><c> the</c><00:12:38.120><c> way</c><00:12:38.279><c> that</c>\nmost common tokenizers and the way that\nmost common tokenizers and the way that you<00:12:38.480><c> train</c><00:12:38.680><c> a</c><00:12:38.959><c> tokenizer</c><00:12:39.959><c> is</c><00:12:40.160><c> that</c><00:12:40.399><c> first</c><00:12:40.600><c> you</c>\nyou train a tokenizer is that first you\nyou train a tokenizer is that first you start<00:12:41.000><c> with</c><00:12:41.120><c> a</c><00:12:41.279><c> very</c><00:12:41.560><c> large</c><00:12:42.000><c> Corpus</c><00:12:42.360><c> of</c><00:12:42.560><c> text</c>\nstart with a very large Corpus of text\nstart with a very large Corpus of text and<00:12:43.120><c> here</c><00:12:43.279><c> I'm</c><00:12:43.440><c> really</c><00:12:43.639><c> not</c><00:12:43.800><c> talking</c><00:12:44.079><c> about</c>\nand here I'm really not talking about\nand here I'm really not talking about training<00:12:44.519><c> a</c><00:12:44.639><c> large</c><00:12:44.839><c> language</c><00:12:45.160><c> model</c><00:12:45.480><c> yet</c><00:12:45.639><c> this</c>\ntraining a large language model yet this\ntraining a large language model yet this is<00:12:45.880><c> purely</c><00:12:46.120><c> for</c><00:12:46.279><c> the</c><00:12:46.399><c> tokenization</c><00:12:47.040><c> step</c><00:12:47.920><c> uh</c>\nis purely for the tokenization step uh\nis purely for the tokenization step uh so<00:12:48.240><c> this</c><00:12:48.360><c> is</c><00:12:48.519><c> my</c><00:12:48.720><c> large</c><00:12:49.199><c> Corpus</c><00:12:49.600><c> of</c><00:12:49.839><c> text</c><00:12:50.240><c> with</c>\nso this is my large Corpus of text with\nso this is my large Corpus of text with these<00:12:50.760><c> five</c><00:12:51.079><c> words</c><00:12:52.079><c> um</c><00:12:52.720><c> then</c><00:12:53.160><c> you</c><00:12:53.720><c> associate</c>\nthese five words um then you associate\nthese five words um then you associate every<00:12:54.760><c> character</c><00:12:55.519><c> in</c><00:12:55.720><c> this</c><00:12:55.920><c> Corpus</c><00:12:56.279><c> of</c><00:12:56.480><c> text</c><00:12:57.240><c> a</c>\nevery character in this Corpus of text a\nevery character in this Corpus of text a different<00:12:57.760><c> token</c><00:12:58.639><c> uh</c><00:12:58.760><c> so</c><00:12:58.920><c> here</c><00:12:59.199><c> I</c><00:12:59.279><c> just</c><00:12:59.399><c> split</c>\ndifferent token uh so here I just split\ndifferent token uh so here I just split up<00:12:59.880><c> every</c><00:13:00.120><c> character</c><00:13:00.480><c> with</c><00:13:00.639><c> a</c><00:13:00.880><c> different</c>\nup every character with a different\nup every character with a different token<00:13:01.920><c> uh</c><00:13:02.320><c> and</c><00:13:02.600><c> I</c><00:13:02.760><c> just</c><00:13:02.959><c> color</c><00:13:03.279><c> coded</c><00:13:03.680><c> all</c><00:13:03.839><c> of</c>\ntoken uh and I just color coded all of\ntoken uh and I just color coded all of those<00:13:04.959><c> tokens</c><00:13:05.959><c> and</c><00:13:06.079><c> then</c><00:13:06.240><c> what</c><00:13:06.320><c> you</c><00:13:06.480><c> do</c><00:13:06.800><c> is</c>\nthose tokens and then what you do is\nthose tokens and then what you do is that<00:13:07.120><c> you</c><00:13:07.279><c> go</c><00:13:07.399><c> through</c><00:13:07.600><c> your</c><00:13:07.800><c> text</c><00:13:08.160><c> and</c><00:13:08.360><c> every</c>\nthat you go through your text and every\nthat you go through your text and every time<00:13:08.800><c> you</c><00:13:08.920><c> see</c><00:13:09.720><c> pairs</c><00:13:10.040><c> of</c><00:13:10.240><c> tokens</c><00:13:10.959><c> that</c><00:13:11.120><c> are</c>\ntime you see pairs of tokens that are\ntime you see pairs of tokens that are very<00:13:11.839><c> common</c><00:13:12.440><c> the</c><00:13:12.600><c> most</c><00:13:12.920><c> common</c><00:13:13.279><c> pair</c><00:13:13.480><c> of</c>\nvery common the most common pair of\nvery common the most common pair of token<00:13:14.120><c> you</c><00:13:14.279><c> just</c><00:13:14.440><c> merge</c><00:13:14.839><c> them</c><00:13:15.240><c> so</c><00:13:15.440><c> here</c><00:13:15.560><c> you</c>\ntoken you just merge them so here you\ntoken you just merge them so here you see<00:13:16.000><c> three</c><00:13:16.240><c> times</c><00:13:17.000><c> the</c><00:13:17.320><c> the</c><00:13:17.920><c> the</c><00:13:18.079><c> tokens</c><00:13:18.760><c> T</c><00:13:19.120><c> and</c>\nsee three times the the the tokens T and\nsee three times the the the tokens T and O<00:13:19.880><c> next</c><00:13:20.079><c> to</c><00:13:20.240><c> each</c><00:13:20.360><c> other</c><00:13:20.639><c> so</c><00:13:20.800><c> you're</c><00:13:20.920><c> just</c>\nO next to each other so you're just\nO next to each other so you're just going<00:13:21.120><c> to</c><00:13:21.240><c> say</c><00:13:21.399><c> this</c><00:13:21.480><c> is</c><00:13:21.560><c> a</c><00:13:21.720><c> new</c><00:13:21.920><c> token</c><00:13:22.760><c> and</c>\ngoing to say this is a new token and\ngoing to say this is a new token and then<00:13:22.959><c> you</c><00:13:23.160><c> continue</c><00:13:23.519><c> you</c><00:13:23.639><c> repeat</c><00:13:24.040><c> that</c><00:13:24.360><c> so</c><00:13:24.519><c> now</c>\nthen you continue you repeat that so now\nthen you continue you repeat that so now you<00:13:24.880><c> have</c><00:13:25.399><c> to</c><00:13:26.399><c> talk</c><00:13:26.959><c> which</c><00:13:27.160><c> happens</c><00:13:27.519><c> three</c>\nyou have to talk which happens three\nyou have to talk which happens three times<00:13:28.519><c> to</c><00:13:29.320><c> with</c><00:13:29.519><c> an</c><00:13:29.680><c> E</c><00:13:30.040><c> that</c><00:13:30.199><c> happens</c><00:13:30.800><c> sorry</c>\ntimes to with an E that happens sorry\ntimes to with an E that happens sorry two<00:13:31.320><c> times</c><00:13:32.199><c> and</c><00:13:32.800><c> an</c><00:13:33.120><c> token</c><00:13:33.639><c> which</c><00:13:33.800><c> happens</c>\ntwo times and an token which happens\ntwo times and an token which happens twice<00:13:34.760><c> and</c><00:13:34.880><c> then</c><00:13:35.079><c> ex</c><00:13:35.800><c> which</c><00:13:35.959><c> also</c><00:13:36.199><c> happen</c>\ntwice and then ex which also happen\ntwice and then ex which also happen twice<00:13:37.160><c> so</c><00:13:37.360><c> this</c><00:13:37.480><c> is</c><00:13:37.680><c> that</c><00:13:38.240><c> if</c><00:13:38.440><c> you</c><00:13:38.600><c> were</c><00:13:38.880><c> to</c>\ntwice so this is that if you were to\ntwice so this is that if you were to train<00:13:39.320><c> a</c><00:13:39.440><c> tokenizer</c><00:13:40.399><c> on</c><00:13:40.639><c> this</c><00:13:40.880><c> Corpus</c><00:13:41.240><c> of</c><00:13:41.440><c> text</c>\ntrain a tokenizer on this Corpus of text\ntrain a tokenizer on this Corpus of text which<00:13:41.880><c> is</c><00:13:42.079><c> very</c><00:13:42.279><c> small</c><00:13:43.000><c> that's</c><00:13:43.199><c> how</c><00:13:43.360><c> you</c><00:13:43.480><c> would</c>\nwhich is very small that's how you would\nwhich is very small that's how you would uh<00:13:43.959><c> finish</c><00:13:44.279><c> with</c><00:13:44.440><c> a</c><00:13:44.560><c> token</c><00:13:45.079><c> with</c><00:13:45.199><c> a</c><00:13:45.360><c> pre</c><00:13:45.680><c> like</c><00:13:45.760><c> a</c>\nuh finish with a token with a pre like a\nuh finish with a token with a pre like a trained<00:13:46.560><c> tokenizer</c><00:13:47.560><c> uh</c><00:13:47.680><c> in</c><00:13:47.839><c> reality</c><00:13:48.240><c> you</c><00:13:48.360><c> do</c>\ntrained tokenizer uh in reality you do\ntrained tokenizer uh in reality you do it<00:13:48.680><c> on</c><00:13:49.040><c> on</c><00:13:49.240><c> much</c><00:13:49.519><c> larger</c><00:13:49.959><c> corpuses</c><00:13:50.399><c> of</c><00:13:50.600><c> text</c><00:13:51.480><c> um</c>\nit on on much larger corpuses of text um\nit on on much larger corpuses of text um and<00:13:52.040><c> this</c><00:13:52.160><c> is</c><00:13:52.279><c> the</c><00:13:52.480><c> real</c><00:13:53.000><c> tokenizer</c><00:13:54.000><c> of</c><00:13:54.399><c> uh</c>\nand this is the real tokenizer of uh\nand this is the real tokenizer of uh actually<00:13:55.240><c> I</c><00:13:55.360><c> think</c><00:13:55.560><c> this</c><00:13:55.639><c> is</c><00:13:55.800><c> gpt3</c><00:13:56.519><c> or</c><00:13:56.759><c> chat</c>\nactually I think this is gpt3 or chat\nactually I think this is gpt3 or chat GPT<00:13:57.920><c> uh</c><00:13:58.040><c> and</c><00:13:58.199><c> here</c><00:13:58.320><c> you</c><00:13:58.440><c> see</c><00:13:58.639><c> how</c><00:13:58.759><c> it</c><00:13:58.880><c> would</c>\nGPT uh and here you see how it would\nGPT uh and here you see how it would actually<00:13:59.399><c> separate</c><00:13:59.839><c> these</c><00:14:00.000><c> words</c><00:14:00.360><c> so</c>\nactually separate these words so\nactually separate these words so basically<00:14:00.880><c> you</c><00:14:01.000><c> see</c><00:14:01.199><c> the</c><00:14:01.320><c> same</c><00:14:01.480><c> thing</c><00:14:01.639><c> as</c><00:14:01.800><c> what</c>\nbasically you see the same thing as what\nbasically you see the same thing as what we<00:14:02.199><c> gave</c><00:14:02.560><c> in</c><00:14:02.680><c> the</c><00:14:02.839><c> previous</c><00:14:03.199><c> example</c><00:14:03.959><c> token</c>\nwe gave in the previous example token\nwe gave in the previous example token becomes<00:14:05.120><c> its</c><00:14:05.279><c> own</c><00:14:05.639><c> token</c><00:14:06.519><c> so</c><00:14:06.880><c> tokenizer</c><00:14:07.880><c> is</c>\nbecomes its own token so tokenizer is\nbecomes its own token so tokenizer is actually<00:14:08.279><c> split</c><00:14:08.639><c> up</c><00:14:08.800><c> into</c><00:14:09.040><c> two</c><00:14:09.320><c> tokens</c><00:14:10.040><c> token</c>\nactually split up into two tokens token\nactually split up into two tokens token and<00:14:11.079><c> iser</c><00:14:12.079><c> um</c><00:14:12.839><c> so</c><00:14:13.079><c> yeah</c><00:14:13.480><c> that's</c><00:14:13.680><c> all</c><00:14:13.880><c> about</c>\nand iser um so yeah that's all about\nand iser um so yeah that's all about tokenizers<00:14:15.160><c> any</c><00:14:15.320><c> questions</c><00:14:15.560><c> on</c><00:14:15.800><c> that</c><00:14:16.279><c> yeah</c>\ntokenizers any questions on that yeah\ntokenizers any questions on that yeah how<00:14:16.680><c> do</c><00:14:16.759><c> you</c><00:14:16.880><c> deal</c><00:14:17.040><c> with</c><00:14:17.199><c> spes</c><00:14:17.560><c> and</c><00:14:17.720><c> how</c><00:14:17.800><c> do</c><00:14:17.880><c> you</c>\nhow do you deal with spes and how do you\nhow do you deal with spes and how do you deal\ndeal\ndeal with<00:14:20.040><c> yeah</c><00:14:20.600><c> so</c><00:14:21.240><c> actually</c><00:14:21.600><c> there's</c><00:14:21.759><c> a</c><00:14:22.120><c> a</c><00:14:22.279><c> step</c>\nwith yeah so actually there's a a step\nwith yeah so actually there's a a step before<00:14:22.800><c> tokenizers</c><00:14:23.560><c> which</c><00:14:23.680><c> is</c><00:14:23.880><c> what</c><00:14:24.000><c> we</c><00:14:24.120><c> call</c>\nbefore tokenizers which is what we call\nbefore tokenizers which is what we call pre-<00:14:24.560><c> tokenizers</c><00:14:25.480><c> which</c><00:14:25.639><c> is</c><00:14:26.320><c> exactly</c><00:14:26.759><c> what</c>\npre- tokenizers which is exactly what\npre- tokenizers which is exactly what you<00:14:27.040><c> just</c><00:14:27.199><c> said</c><00:14:27.880><c> uh</c><00:14:27.959><c> so</c><00:14:28.160><c> this</c><00:14:28.279><c> is</c><00:14:28.519><c> mostly</c>\nyou just said uh so this is mostly\nyou just said uh so this is mostly in<00:14:29.800><c> theory</c><00:14:30.160><c> there's</c><00:14:30.360><c> no</c><00:14:30.560><c> reason</c><00:14:30.800><c> to</c><00:14:31.000><c> deal</c><00:14:31.240><c> with</c>\nin theory there's no reason to deal with\nin theory there's no reason to deal with spaces<00:14:32.279><c> and</c><00:14:32.720><c> punctuation</c><00:14:33.720><c> separately</c><00:14:34.240><c> you</c>\nspaces and punctuation separately you\nspaces and punctuation separately you could<00:14:34.519><c> just</c><00:14:34.680><c> say</c><00:14:35.000><c> every</c><00:14:35.320><c> space</c><00:14:35.680><c> gets</c><00:14:35.920><c> its</c><00:14:36.079><c> own</c>\ncould just say every space gets its own\ncould just say every space gets its own token<00:14:37.120><c> every</c><00:14:38.040><c> um</c><00:14:38.920><c> uh</c><00:14:39.040><c> punctuation</c><00:14:39.560><c> get</c><00:14:39.759><c> its</c>\ntoken every um uh punctuation get its\ntoken every um uh punctuation get its own<00:14:40.160><c> token</c><00:14:40.639><c> and</c><00:14:40.759><c> you</c><00:14:40.839><c> can</c><00:14:41.000><c> just</c><00:14:41.160><c> do</c><00:14:41.360><c> all</c><00:14:41.519><c> the</c>\nown token and you can just do all the\nown token and you can just do all the merging<00:14:42.399><c> the</c><00:14:42.600><c> problem</c><00:14:42.880><c> is</c><00:14:43.079><c> that</c><00:14:43.240><c> so</c><00:14:43.440><c> there's</c>\nmerging the problem is that so there's\nmerging the problem is that so there's an<00:14:43.720><c> efficiency</c><00:14:44.240><c> question</c><00:14:44.959><c> actually</c><00:14:45.360><c> training</c>\nan efficiency question actually training\nan efficiency question actually training these<00:14:45.839><c> tokenizes</c><00:14:46.480><c> takes</c><00:14:46.680><c> a</c><00:14:46.920><c> long</c><00:14:47.360><c> time</c><00:14:48.120><c> uh</c><00:14:48.240><c> so</c>\nthese tokenizes takes a long time uh so\nthese tokenizes takes a long time uh so you<00:14:48.639><c> better</c><00:14:48.959><c> off</c><00:14:49.120><c> because</c><00:14:49.279><c> you</c><00:14:49.399><c> have</c><00:14:49.519><c> to</c>\nyou better off because you have to\nyou better off because you have to consider<00:14:50.519><c> every</c><00:14:50.880><c> pair</c><00:14:51.079><c> of</c><00:14:51.240><c> token</c><00:14:51.880><c> so</c><00:14:52.040><c> what</c><00:14:52.160><c> you</c>\nconsider every pair of token so what you\nconsider every pair of token so what you end<00:14:52.480><c> up</c><00:14:52.639><c> doing</c><00:14:52.880><c> is</c><00:14:53.000><c> saying</c><00:14:53.279><c> if</c><00:14:53.399><c> there's</c><00:14:53.519><c> a</c>\nend up doing is saying if there's a\nend up doing is saying if there's a space<00:14:54.240><c> this</c><00:14:54.320><c> is</c><00:14:54.560><c> very</c><00:14:54.800><c> like</c><00:14:54.959><c> pre-</c><00:14:55.199><c> tokenizes</c>\nspace this is very like pre- tokenizes\nspace this is very like pre- tokenizes are<00:14:55.839><c> very</c><00:14:56.000><c> English</c><00:14:56.399><c> specific</c><00:14:57.040><c> you</c><00:14:57.199><c> say</c><00:14:57.399><c> if</c>\nare very English specific you say if\nare very English specific you say if there's<00:14:57.680><c> a</c><00:14:57.839><c> space</c><00:14:58.360><c> we're</c><00:14:58.519><c> not</c><00:14:58.680><c> going</c><00:14:58.759><c> to</c><00:14:59.120><c> start</c>\nthere's a space we're not going to start\nthere's a space we're not going to start looking<00:14:59.600><c> at</c><00:14:59.839><c> the</c><00:15:00.040><c> the</c><00:15:00.120><c> token</c><00:15:00.440><c> that</c><00:15:00.600><c> came</c>\nlooking at the the token that came\nlooking at the the token that came before<00:15:01.519><c> and</c><00:15:01.639><c> the</c><00:15:01.759><c> token</c><00:15:02.079><c> that</c><00:15:02.320><c> came</c>\nbefore and the token that came\nbefore and the token that came afterwards<00:15:03.160><c> so</c><00:15:03.279><c> you're</c><00:15:03.399><c> not</c><00:15:03.639><c> merging</c><00:15:04.240><c> in</c>\nafterwards so you're not merging in\nafterwards so you're not merging in between<00:15:05.320><c> spaces</c><00:15:06.079><c> but</c><00:15:06.240><c> this</c><00:15:06.360><c> is</c><00:15:06.560><c> just</c><00:15:06.759><c> like</c><00:15:07.000><c> a</c>\nbetween spaces but this is just like a\nbetween spaces but this is just like a optimiz<00:15:08.399><c> like</c><00:15:08.519><c> a</c><00:15:08.680><c> computation</c><00:15:09.240><c> optimization</c>\noptimiz like a computation optimization\noptimiz like a computation optimization you<00:15:10.199><c> could</c><00:15:10.639><c> theoretically</c><00:15:11.199><c> just</c><00:15:11.399><c> deal</c><00:15:11.639><c> with</c>\nyou could theoretically just deal with\nyou could theoretically just deal with it<00:15:12.519><c> um</c><00:15:12.759><c> the</c><00:15:12.839><c> same</c><00:15:13.040><c> way</c><00:15:13.199><c> as</c><00:15:13.279><c> you</c><00:15:13.399><c> deal</c><00:15:13.600><c> with</c><00:15:13.759><c> any</c>\nit um the same way as you deal with any\nit um the same way as you deal with any other<00:15:14.320><c> character</c><00:15:15.320><c> and</c><00:15:15.959><c> yeah</c><00:15:16.399><c> when</c><00:15:16.480><c> you</c><00:15:16.680><c> merge</c>\nother character and yeah when you merge\nother character and yeah when you merge tokens<00:15:17.440><c> do</c><00:15:17.519><c> you</c><00:15:17.839><c> delete</c><00:15:18.360><c> the</c><00:15:18.480><c> tokens</c><00:15:18.839><c> that</c><00:15:18.920><c> you</c>\ntokens do you delete the tokens that you\ntokens do you delete the tokens that you merged<00:15:19.440><c> away</c><00:15:19.720><c> or</c><00:15:19.959><c> do</c><00:15:20.040><c> you</c><00:15:20.279><c> keep</c><00:15:20.880><c> the</c><00:15:21.160><c> the</c>\nmerged away or do you keep the the\nmerged away or do you keep the the smaller<00:15:21.639><c> tokens</c><00:15:22.000><c> that</c><00:15:22.240><c> merge</c><00:15:22.800><c> um</c><00:15:23.160><c> you</c>\nsmaller tokens that merge um you\nsmaller tokens that merge um you actually<00:15:23.759><c> keep</c><00:15:24.040><c> the</c><00:15:24.240><c> smaller</c><00:15:24.720><c> tokens</c><00:15:25.240><c> I</c><00:15:25.320><c> mean</c>\nactually keep the smaller tokens I mean\nactually keep the smaller tokens I mean in<00:15:25.600><c> reality</c><00:15:25.920><c> it</c><00:15:26.000><c> doesn't</c><00:15:26.279><c> matter</c><00:15:26.560><c> much</c>\nin reality it doesn't matter much\nin reality it doesn't matter much because<00:15:27.959><c> um</c><00:15:29.040><c> usually</c><00:15:29.800><c> on</c><00:15:30.240><c> large</c><00:15:30.560><c> Corpus</c><00:15:30.920><c> of</c>\nbecause um usually on large Corpus of\nbecause um usually on large Corpus of text<00:15:31.279><c> you</c><00:15:31.360><c> will</c><00:15:31.519><c> have</c><00:15:31.720><c> actually</c><00:15:31.959><c> everything</c>\ntext you will have actually everything\ntext you will have actually everything uh<00:15:33.120><c> but</c><00:15:33.240><c> you</c><00:15:33.399><c> usually</c><00:15:33.680><c> keep</c><00:15:33.920><c> the</c><00:15:34.040><c> small</c><00:15:34.240><c> ones</c>\nuh but you usually keep the small ones\nuh but you usually keep the small ones and<00:15:34.560><c> the</c><00:15:34.639><c> reason</c><00:15:34.880><c> why</c><00:15:34.959><c> you</c><00:15:35.040><c> want</c><00:15:35.160><c> to</c><00:15:35.279><c> do</c><00:15:35.440><c> that</c>\nand the reason why you want to do that\nand the reason why you want to do that is<00:15:35.759><c> because</c><00:15:36.000><c> if</c><00:15:36.240><c> in</c><00:15:36.480><c> case</c><00:15:36.720><c> there's</c><00:15:37.240><c> as</c><00:15:37.360><c> we</c><00:15:37.519><c> said</c>\nis because if in case there's as we said\nis because if in case there's as we said before<00:15:38.440><c> you</c><00:15:38.680><c> have</c><00:15:38.920><c> some</c><00:15:39.360><c> um</c><00:15:39.839><c> some</c><00:15:40.040><c> grammatical</c>\nbefore you have some um some grammatical\nbefore you have some um some grammatical mistakes<00:15:40.839><c> so</c><00:15:41.000><c> some</c><00:15:41.120><c> typos</c><00:15:41.720><c> you</c><00:15:41.839><c> still</c><00:15:42.040><c> want</c><00:15:42.120><c> to</c>\nmistakes so some typos you still want to\nmistakes so some typos you still want to be<00:15:42.399><c> able</c><00:15:42.600><c> to</c><00:15:42.800><c> represent</c><00:15:43.480><c> these</c><00:15:43.680><c> words</c><00:15:44.000><c> by</c>\nbe able to represent these words by\nbe able to represent these words by character<00:15:45.600><c> um</c><00:15:46.560><c> so</c><00:15:47.040><c> yeah</c><00:15:48.040><c> yes</c><00:15:48.839><c> are</c><00:15:49.000><c> the</c><00:15:49.399><c> tokens</c>\ncharacter um so yeah yes are the tokens\ncharacter um so yeah yes are the tokens unique<00:15:51.199><c> so</c><00:15:51.600><c> I</c><00:15:51.680><c> mean</c><00:15:52.360><c> say</c><00:15:52.600><c> in</c><00:15:52.720><c> this</c><00:15:52.880><c> case</c><00:15:53.160><c> T</c><00:15:53.560><c> Ken</c>\nunique so I mean say in this case T Ken\nunique so I mean say in this case T Ken is<00:15:54.360><c> there</c><00:15:54.519><c> only</c><00:15:54.759><c> one</c><00:15:55.000><c> occurrence</c><00:15:55.399><c> or</c><00:15:55.720><c> could</c><00:15:56.240><c> do</c>\nis there only one occurrence or could do\nis there only one occurrence or could do you<00:15:56.600><c> need</c><00:15:56.800><c> to</c><00:15:57.800><c> leave</c><00:15:58.199><c> multiple</c><00:15:58.600><c> occurr</c><00:15:59.160><c> so</c>\nyou need to leave multiple occurr so\nyou need to leave multiple occurr so they<00:15:59.440><c> could</c><00:15:59.639><c> have</c><00:16:00.160><c> take</c><00:16:00.319><c> on</c><00:16:00.560><c> different</c>\nthey could have take on different\nthey could have take on different meanings<00:16:01.240><c> or</c><00:16:01.399><c> something</c><00:16:01.959><c> oh</c><00:16:02.199><c> oh</c><00:16:02.360><c> I</c><00:16:02.440><c> see</c><00:16:02.600><c> what</c>\nmeanings or something oh oh I see what\nmeanings or something oh oh I see what you<00:16:02.959><c> say</c><00:16:03.199><c> no</c><00:16:03.480><c> no</c><00:16:03.639><c> it's</c><00:16:03.959><c> every</c><00:16:04.240><c> token</c><00:16:04.880><c> has</c><00:16:05.079><c> its</c>\nyou say no no it's every token has its\nyou say no no it's every token has its own<00:16:06.319><c> uh</c><00:16:06.600><c> unique</c><00:16:07.240><c> ID</c><00:16:08.240><c> um</c><00:16:08.759><c> so</c><00:16:09.199><c> a</c><00:16:09.399><c> usual</c><00:16:10.160><c> this</c><00:16:10.240><c> is</c><00:16:10.319><c> a</c>\nown uh unique ID um so a usual this is a\nown uh unique ID um so a usual this is a great<00:16:10.680><c> question</c><00:16:10.959><c> for</c><00:16:11.120><c> example</c><00:16:11.399><c> if</c><00:16:11.480><c> you</c><00:16:11.600><c> think</c>\ngreat question for example if you think\ngreat question for example if you think about<00:16:12.319><c> a</c><00:16:12.680><c> bank</c><00:16:13.199><c> which</c><00:16:13.360><c> could</c><00:16:13.480><c> be</c><00:16:13.639><c> bank</c><00:16:13.880><c> for</c>\nabout a bank which could be bank for\nabout a bank which could be bank for like<00:16:14.199><c> money</c><00:16:14.440><c> or</c><00:16:14.600><c> bank</c><00:16:14.880><c> like</c><00:16:15.120><c> water</c><00:16:16.040><c> um</c><00:16:16.440><c> it</c><00:16:16.639><c> will</c>\nlike money or bank like water um it will\nlike money or bank like water um it will have<00:16:17.040><c> the</c><00:16:17.199><c> same</c><00:16:17.440><c> token</c><00:16:18.120><c> but</c><00:16:18.279><c> the</c><00:16:18.440><c> model</c><00:16:18.800><c> will</c>\nhave the same token but the model will\nhave the same token but the model will learn<00:16:19.279><c> the</c><00:16:19.399><c> Transformer</c><00:16:19.959><c> will</c><00:16:20.160><c> learn</c><00:16:20.639><c> that</c>\nlearn the Transformer will learn that\nlearn the Transformer will learn that based<00:16:21.160><c> on</c><00:16:21.279><c> the</c><00:16:21.399><c> words</c><00:16:21.680><c> that</c><00:16:21.800><c> are</c><00:16:21.959><c> around</c><00:16:22.240><c> it</c><00:16:22.880><c> it</c>\nbased on the words that are around it it\nbased on the words that are around it it should<00:16:23.519><c> associate</c><00:16:24.199><c> that</c><00:16:24.959><c> I'm</c><00:16:25.079><c> saying</c><00:16:25.279><c> I'm</c>\nshould associate that I'm saying I'm\nshould associate that I'm saying I'm being<00:16:25.600><c> very</c><00:16:25.800><c> high</c><00:16:26.040><c> wavy</c><00:16:26.399><c> here</c><00:16:26.560><c> but</c><00:16:26.800><c> associate</c>\nbeing very high wavy here but associate\nbeing very high wavy here but associate that<00:16:27.440><c> with</c><00:16:27.639><c> the</c><00:16:28.000><c> with</c><00:16:28.120><c> a</c><00:16:28.560><c> with</c><00:16:28.720><c> a</c>\nthat with the with a with a\nthat with the with a with a representation<00:16:30.279><c> that</c><00:16:30.440><c> is</c><00:16:30.639><c> either</c><00:16:30.920><c> more</c><00:16:31.199><c> like</c>\nrepresentation that is either more like\nrepresentation that is either more like the<00:16:31.800><c> bank</c><00:16:32.279><c> money</c><00:16:32.639><c> side</c><00:16:32.880><c> or</c><00:16:33.040><c> the</c><00:16:33.199><c> Bank</c><00:16:33.680><c> water</c>\nthe bank money side or the Bank water\nthe bank money side or the Bank water side<00:16:34.759><c> um</c><00:16:34.920><c> but</c><00:16:35.040><c> that's</c><00:16:35.160><c> a</c><00:16:35.279><c> Transformer</c><00:16:35.759><c> that</c>\nside um but that's a Transformer that\nside um but that's a Transformer that does<00:16:36.160><c> that</c><00:16:36.360><c> it's</c><00:16:36.440><c> not</c><00:16:36.600><c> a</c>\ndoes that it's not a\ndoes that it's not a tokenizer<00:16:38.319><c> yes</c><00:16:39.279><c> yeah</c><00:16:39.399><c> so</c><00:16:39.519><c> you</c><00:16:39.639><c> mentioned</c>\ntokenizer yes yeah so you mentioned\ntokenizer yes yeah so you mentioned during<00:16:40.279><c> tokenization</c><00:16:41.040><c> keep</c><00:16:41.240><c> the</c><00:16:41.360><c> smaller</c>\nduring tokenization keep the smaller\nduring tokenization keep the smaller tokens<00:16:42.120><c> you</c><00:16:42.240><c> started</c><00:16:42.600><c> with</c><00:16:42.880><c> right</c><00:16:43.600><c> like</c><00:16:44.199><c> if</c>\ntokens you started with right like if\ntokens you started with right like if you<00:16:44.399><c> start</c><00:16:44.600><c> with</c><00:16:44.720><c> a</c><00:16:44.920><c> t</c><00:16:45.279><c> you</c><00:16:45.440><c> keep</c><00:16:45.639><c> the</c><00:16:45.800><c> T</c><00:16:46.120><c> and</c>\nyou start with a t you keep the T and\nyou start with a t you keep the T and then<00:16:46.440><c> you</c><00:16:46.680><c> build</c><00:16:46.920><c> your</c><00:16:47.040><c> tokenizer</c><00:16:47.680><c> to</c><00:16:47.800><c> the</c>\nthen you build your tokenizer to the\nthen you build your tokenizer to the that<00:16:48.240><c> you</c><00:16:48.360><c> can</c><00:16:48.519><c> now</c><00:16:48.680><c> in</c><00:16:49.120><c> token</c><00:16:49.839><c> so</c><00:16:50.079><c> let's</c><00:16:50.279><c> say</c>\nthat you can now in token so let's say\nthat you can now in token so let's say maybe<00:16:51.120><c> you</c><00:16:51.240><c> didn't</c><00:16:51.399><c> train</c><00:16:51.639><c> on</c><00:16:51.839><c> token</c><00:16:52.160><c> but</c><00:16:52.319><c> like</c>\nmaybe you didn't train on token but like\nmaybe you didn't train on token but like in<00:16:52.600><c> your</c><00:16:52.800><c> data</c><00:16:53.160><c> you</c><00:16:53.240><c> are</c><00:16:53.360><c> trying</c><00:16:53.600><c> to</c><00:16:53.880><c> encode</c>\nin your data you are trying to encode\nin your data you are trying to encode token<00:16:55.279><c> so</c><00:16:55.519><c> how</c><00:16:55.680><c> does</c><00:16:56.000><c> the</c><00:16:56.240><c> tokenizer</c><00:16:56.880><c> know</c><00:16:57.120><c> to</c>\ntoken so how does the tokenizer know to\ntoken so how does the tokenizer know to encode<00:16:57.759><c> it</c><00:16:58.000><c> with</c><00:16:58.279><c> token</c><00:16:58.600><c> or</c>\nencode it with token or\nencode it with token or a<00:17:00.160><c> great</c><00:17:00.360><c> question</c><00:17:00.720><c> you</c><00:17:00.920><c> basically</c><00:17:01.360><c> when</c><00:17:01.519><c> you</c>\na great question you basically when you\na great question you basically when you so<00:17:02.199><c> when</c><00:17:02.279><c> you</c><00:17:02.440><c> tokenize</c><00:17:02.959><c> so</c><00:17:03.120><c> that's</c><00:17:03.360><c> after</c>\nso when you tokenize so that's after\nso when you tokenize so that's after training<00:17:03.920><c> of</c><00:17:04.039><c> the</c><00:17:04.160><c> tokenizer</c><00:17:04.679><c> when</c><00:17:04.760><c> you</c>\ntraining of the tokenizer when you\ntraining of the tokenizer when you actually<00:17:05.400><c> apply</c><00:17:05.720><c> the</c><00:17:05.880><c> tokenizer</c><00:17:06.640><c> you</c>\nactually apply the tokenizer you\nactually apply the tokenizer you basically<00:17:07.360><c> always</c><00:17:07.720><c> choose</c><00:17:08.079><c> the</c><00:17:08.559><c> largest</c><00:17:09.559><c> uh</c>\nbasically always choose the largest uh\nbasically always choose the largest uh token<00:17:10.160><c> that</c><00:17:10.240><c> you</c><00:17:10.360><c> can</c><00:17:10.520><c> apply</c><00:17:11.480><c> uh</c><00:17:11.600><c> so</c><00:17:11.760><c> if</c><00:17:11.839><c> you</c>\ntoken that you can apply uh so if you\ntoken that you can apply uh so if you can<00:17:12.079><c> do</c><00:17:12.240><c> token</c><00:17:12.559><c> you</c><00:17:12.640><c> will</c><00:17:12.799><c> never</c><00:17:13.000><c> do</c><00:17:13.160><c> T</c><00:17:13.640><c> you</c>\ncan do token you will never do T you\ncan do token you will never do T you will<00:17:13.919><c> always</c><00:17:14.120><c> do</c><00:17:14.559><c> token</c><00:17:15.559><c> um</c><00:17:16.199><c> but</c><00:17:16.360><c> there's</c>\nwill always do token um but there's\nwill always do token um but there's actually<00:17:17.000><c> so</c><00:17:17.520><c> people</c><00:17:17.760><c> don't</c><00:17:17.959><c> usually</c><00:17:18.240><c> talk</c>\nactually so people don't usually talk\nactually so people don't usually talk that<00:17:18.600><c> much</c><00:17:18.760><c> about</c><00:17:18.959><c> tokenizers</c><00:17:19.720><c> but</c><00:17:20.039><c> uh</c>\nthat much about tokenizers but uh\nthat much about tokenizers but uh there's<00:17:20.600><c> a</c><00:17:20.720><c> lot</c><00:17:20.880><c> of</c><00:17:21.480><c> of</c><00:17:21.760><c> computational</c>\nthere's a lot of of computational\nthere's a lot of of computational benefits<00:17:23.280><c> uh</c><00:17:23.480><c> or</c><00:17:23.679><c> computational</c><00:17:24.240><c> tricks</c><00:17:24.559><c> that</c>\nbenefits uh or computational tricks that\nbenefits uh or computational tricks that you<00:17:24.760><c> can</c><00:17:24.919><c> do</c><00:17:25.240><c> for</c><00:17:25.439><c> making</c><00:17:25.720><c> these</c><00:17:25.880><c> things</c>\nyou can do for making these things\nyou can do for making these things faster<00:17:27.160><c> uh</c><00:17:27.240><c> so</c><00:17:27.400><c> I</c><00:17:27.520><c> really</c><00:17:27.679><c> don't</c><00:17:27.880><c> think</c><00:17:28.079><c> we</c><00:17:28.280><c> and</c>\nfaster uh so I really don't think we and\nfaster uh so I really don't think we and honestly<00:17:29.080><c> I</c><00:17:29.160><c> think</c><00:17:29.280><c> a</c><00:17:29.360><c> lot</c><00:17:29.440><c> of</c><00:17:29.559><c> people</c><00:17:29.760><c> think</c>\nhonestly I think a lot of people think\nhonestly I think a lot of people think that<00:17:30.039><c> we</c><00:17:30.160><c> should</c><00:17:30.360><c> just</c><00:17:30.559><c> get</c><00:17:30.880><c> away</c><00:17:31.120><c> from</c>\nthat we should just get away from\nthat we should just get away from tokenizers<00:17:32.679><c> um</c><00:17:33.120><c> and</c><00:17:33.280><c> just</c><00:17:33.440><c> kind</c><00:17:33.559><c> of</c><00:17:33.679><c> tokenize</c>\ntokenizers um and just kind of tokenize\ntokenizers um and just kind of tokenize character<00:17:34.559><c> by</c><00:17:34.720><c> character</c><00:17:35.480><c> or</c><00:17:35.679><c> bites</c><00:17:36.000><c> by</c><00:17:36.200><c> bites</c>\ncharacter by character or bites by bites\ncharacter by character or bites by bites uh<00:17:37.160><c> but</c><00:17:37.280><c> as</c><00:17:37.400><c> I</c><00:17:37.520><c> said</c><00:17:37.760><c> right</c><00:17:37.880><c> now</c><00:17:38.039><c> there's</c><00:17:38.240><c> this</c>\nuh but as I said right now there's this\nuh but as I said right now there's this issue<00:17:38.679><c> of</c><00:17:38.840><c> like</c><00:17:39.000><c> length</c><00:17:39.799><c> uh</c><00:17:39.919><c> but</c><00:17:40.080><c> maybe</c><00:17:40.360><c> one</c>\nissue of like length uh but maybe one\nissue of like length uh but maybe one day<00:17:40.760><c> like</c><00:17:40.880><c> in</c><00:17:41.039><c> five</c><00:17:41.240><c> or</c><00:17:41.400><c> 10</c><00:17:41.640><c> years</c><00:17:42.280><c> we</c><00:17:42.360><c> will</c>\nday like in five or 10 years we will\nday like in five or 10 years we will have<00:17:42.679><c> different</c><00:17:42.960><c> architectures</c><00:17:43.440><c> that</c><00:17:43.559><c> don't</c>\nhave different architectures that don't\nhave different architectures that don't scale<00:17:44.039><c> quadratically</c><00:17:44.600><c> with</c><00:17:44.760><c> the</c><00:17:45.039><c> length</c><00:17:45.280><c> of</c>\nscale quadratically with the length of\nscale quadratically with the length of the<00:17:45.520><c> sequence</c><00:17:46.120><c> and</c><00:17:46.440><c> uh</c><00:17:46.600><c> maybe</c><00:17:47.240><c> we'll</c><00:17:48.240><c> um</c><00:17:49.080><c> yeah</c>\nthe sequence and uh maybe we'll um yeah\nthe sequence and uh maybe we'll um yeah move<00:17:49.600><c> away</c><00:17:49.760><c> from</c><00:17:50.000><c> tokenizes</c><00:17:51.000><c> so</c><00:17:51.280><c> can</c><00:17:51.400><c> you</c>\nmove away from tokenizes so can you\nmove away from tokenizes so can you share<00:17:51.840><c> with</c><00:17:51.960><c> us</c><00:17:52.120><c> the</c><00:17:52.280><c> drawback</c><00:17:53.240><c> why</c><00:17:53.360><c> do</c><00:17:53.520><c> people</c>\nshare with us the drawback why do people\nshare with us the drawback why do people want<00:17:53.799><c> to</c><00:17:53.960><c> move</c><00:17:54.160><c> away</c><00:17:54.360><c> from</c><00:17:54.480><c> the</c><00:17:54.679><c> tokenizer</c><00:17:55.679><c> oh</c>\nwant to move away from the tokenizer oh\nwant to move away from the tokenizer oh um<00:17:57.760><c> yeah</c><00:17:58.000><c> so</c><00:17:58.320><c> think</c>\num yeah so think\num yeah so think one<00:18:00.240><c> good</c><00:18:00.480><c> example</c><00:18:01.360><c> is</c><00:18:02.240><c> uh</c><00:18:02.640><c> math</c><00:18:03.559><c> if</c><00:18:03.640><c> you</c><00:18:03.799><c> think</c>\none good example is uh math if you think\none good example is uh math if you think about<00:18:04.240><c> math</c><00:18:04.760><c> actually</c><00:18:05.200><c> numbers</c><00:18:05.640><c> right</c><00:18:05.799><c> now</c>\nabout math actually numbers right now\nabout math actually numbers right now are<00:18:06.159><c> not</c><00:18:06.320><c> tokenized</c><00:18:07.159><c> so</c><00:18:07.320><c> for</c><00:18:07.440><c> example</c><00:18:07.840><c> 327</c>\nare not tokenized so for example 327\nare not tokenized so for example 327 might<00:18:09.039><c> have</c><00:18:09.240><c> its</c><00:18:09.360><c> own</c><00:18:09.600><c> token</c><00:18:10.400><c> which</c><00:18:10.559><c> means</c>\nmight have its own token which means\nmight have its own token which means that<00:18:11.000><c> models</c><00:18:11.559><c> when</c><00:18:11.760><c> they</c><00:18:11.960><c> see</c><00:18:12.440><c> numbers</c><00:18:13.120><c> they</c>\nthat models when they see numbers they\nthat models when they see numbers they don't<00:18:13.480><c> see</c><00:18:13.720><c> them</c><00:18:13.960><c> the</c><00:18:14.200><c> same</c><00:18:14.400><c> way</c><00:18:14.559><c> as</c><00:18:14.720><c> we</c><00:18:14.919><c> do</c><00:18:15.559><c> and</c>\ndon't see them the same way as we do and\ndon't see them the same way as we do and this<00:18:15.919><c> is</c><00:18:16.039><c> very</c><00:18:16.280><c> annoying</c><00:18:16.679><c> because</c><00:18:16.960><c> what</c><00:18:17.200><c> I</c>\nthis is very annoying because what I\nthis is very annoying because what I mean<00:18:17.480><c> the</c><00:18:17.600><c> reason</c><00:18:17.960><c> why</c><00:18:18.080><c> we</c><00:18:18.240><c> can</c><00:18:18.520><c> kind</c><00:18:18.640><c> of</c>\nmean the reason why we can kind of\nmean the reason why we can kind of generalize<00:18:19.240><c> with</c><00:18:19.400><c> math</c><00:18:19.960><c> is</c><00:18:20.120><c> because</c><00:18:20.320><c> we</c><00:18:20.440><c> can</c>\ngeneralize with math is because we can\ngeneralize with math is because we can deal<00:18:20.840><c> with</c><00:18:21.039><c> every</c><00:18:21.480><c> every</c><00:18:21.720><c> letter</c><00:18:22.080><c> separately</c>\ndeal with every every letter separately\ndeal with every every letter separately and<00:18:22.679><c> we</c><00:18:22.880><c> can</c><00:18:23.039><c> then</c><00:18:23.200><c> do</c><00:18:23.480><c> composition</c><00:18:24.280><c> where</c><00:18:24.440><c> you</c>\nand we can then do composition where you\nand we can then do composition where you know<00:18:24.720><c> that</c><00:18:24.880><c> basically</c><00:18:25.159><c> if</c><00:18:25.280><c> you</c><00:18:25.520><c> add</c><00:18:25.840><c> stuff</c>\nknow that basically if you add stuff\nknow that basically if you add stuff it's<00:18:26.240><c> just</c><00:18:26.360><c> the</c><00:18:26.440><c> same</c><00:18:26.640><c> thing</c><00:18:26.799><c> as</c><00:18:26.960><c> adding</c><00:18:27.440><c> every</c>\nit's just the same thing as adding every\nit's just the same thing as adding every one<00:18:28.280><c> separately</c><00:18:28.919><c> plus</c><00:18:29.200><c> like</c><00:18:29.320><c> whatever</c><00:18:29.600><c> the</c>\none separately plus like whatever the\none separately plus like whatever the unit<00:18:30.000><c> that</c><00:18:30.120><c> you</c><00:18:30.280><c> add</c><00:18:30.880><c> so</c><00:18:31.080><c> they</c><00:18:31.200><c> can</c><00:18:31.400><c> do</c><00:18:31.640><c> that</c><00:18:32.400><c> um</c>\nunit that you add so they can do that um\nunit that you add so they can do that um so<00:18:32.919><c> then</c><00:18:33.039><c> you</c><00:18:33.159><c> have</c><00:18:33.320><c> to</c><00:18:33.480><c> do</c><00:18:33.679><c> like</c><00:18:33.880><c> special</c>\nso then you have to do like special\nso then you have to do like special tokenization<00:18:35.360><c> and</c><00:18:35.799><c> like</c><00:18:36.159><c> one</c><00:18:36.320><c> of</c><00:18:36.440><c> the</c><00:18:36.600><c> big</c>\ntokenization and like one of the big\ntokenization and like one of the big changes<00:18:37.280><c> that</c><00:18:37.440><c> GPT</c><00:18:38.280><c> 4</c><00:18:38.880><c> did</c><00:18:39.640><c> uh</c><00:18:39.760><c> is</c><00:18:39.960><c> changing</c>\nchanges that GPT 4 did uh is changing\nchanges that GPT 4 did uh is changing the<00:18:40.679><c> way</c><00:18:40.840><c> that</c><00:18:40.960><c> they</c><00:18:41.159><c> tokenize</c><00:18:42.159><c> uh</c><00:18:42.320><c> code</c><00:18:42.919><c> so</c>\nthe way that they tokenize uh code so\nthe way that they tokenize uh code so for<00:18:43.240><c> example</c><00:18:43.840><c> uh</c><00:18:43.960><c> if</c><00:18:44.039><c> you</c><00:18:44.159><c> have</c><00:18:44.320><c> code</c><00:18:44.799><c> you</c><00:18:44.919><c> know</c>\nfor example uh if you have code you know\nfor example uh if you have code you know you<00:18:45.159><c> have</c><00:18:45.320><c> like</c><00:18:45.559><c> often</c><00:18:46.000><c> in</c><00:18:46.200><c> Python</c><00:18:46.520><c> these</c><00:18:46.640><c> four</c>\nyou have like often in Python these four\nyou have like often in Python these four spaces<00:18:47.320><c> at</c><00:18:47.400><c> the</c><00:18:47.520><c> beginning</c><00:18:48.200><c> those</c><00:18:48.360><c> were</c><00:18:48.600><c> dealt</c>\nspaces at the beginning those were dealt\nspaces at the beginning those were dealt with<00:18:49.799><c> uh</c><00:18:49.960><c> kind</c><00:18:50.120><c> of</c><00:18:50.520><c> strangely</c><00:18:51.159><c> before</c><00:18:52.080><c> um</c><00:18:52.280><c> and</c>\nwith uh kind of strangely before um and\nwith uh kind of strangely before um and as<00:18:52.480><c> a</c><00:18:52.600><c> result</c><00:18:53.000><c> like</c><00:18:53.120><c> the</c><00:18:53.200><c> model</c><00:18:53.559><c> couldn't</c>\nas a result like the model couldn't\nas a result like the model couldn't really<00:18:54.520><c> understand</c><00:18:55.280><c> uh</c><00:18:55.440><c> how</c><00:18:55.640><c> to</c><00:18:56.080><c> deal</c><00:18:56.320><c> with</c>\nreally understand uh how to deal with\nreally understand uh how to deal with code<00:18:57.320><c> uh</c><00:18:57.440><c> so</c><00:18:57.679><c> so</c><00:18:57.840><c> toiz</c><00:18:58.360><c> actually</c><00:18:58.919><c> a</c><00:18:59.039><c> lot</c><00:18:59.919><c> um</c>\ncode uh so so toiz actually a lot um\ncode uh so so toiz actually a lot um okay<00:19:01.520><c> so</c><00:19:01.760><c> I'll</c><00:19:01.919><c> move</c><00:19:02.120><c> on</c><00:19:02.919><c> right</c><00:19:03.039><c> now</c><00:19:03.200><c> but</c><00:19:03.320><c> we</c>\nokay so I'll move on right now but we\nokay so I'll move on right now but we can<00:19:03.520><c> come</c><00:19:03.679><c> back</c><00:19:03.840><c> later</c><00:19:04.080><c> on</c><00:19:04.200><c> token</c><00:19:04.960><c> Isis</c><00:19:05.960><c> great</c>\ncan come back later on token Isis great\ncan come back later on token Isis great so<00:19:06.640><c> we</c><00:19:06.799><c> talked</c><00:19:07.000><c> about</c><00:19:07.120><c> the</c><00:19:07.240><c> task</c><00:19:07.480><c> the</c><00:19:07.600><c> L</c><00:19:07.840><c> the</c>\nso we talked about the task the L the\nso we talked about the task the L the tokenizer<00:19:08.880><c> let's</c><00:19:09.080><c> talk</c><00:19:09.240><c> a</c><00:19:09.320><c> little</c><00:19:09.440><c> bit</c><00:19:09.559><c> about</c>\ntokenizer let's talk a little bit about\ntokenizer let's talk a little bit about evaluation<00:19:11.360><c> uh</c><00:19:11.480><c> so</c><00:19:11.640><c> the</c><00:19:11.760><c> way</c><00:19:11.960><c> that</c><00:19:12.080><c> LMS</c><00:19:12.480><c> are</c>\nevaluation uh so the way that LMS are\nevaluation uh so the way that LMS are usually<00:19:12.919><c> evaluated</c><00:19:13.720><c> is</c><00:19:13.880><c> what</c><00:19:14.000><c> we</c><00:19:14.159><c> call</c><00:19:14.440><c> is</c>\nusually evaluated is what we call is\nusually evaluated is what we call is using<00:19:15.080><c> what</c><00:19:15.200><c> we</c><00:19:15.320><c> call</c><00:19:15.679><c> perplexity</c><00:19:16.679><c> um</c><00:19:16.880><c> at</c><00:19:16.960><c> a</c>\nusing what we call perplexity um at a\nusing what we call perplexity um at a high<00:19:17.320><c> level</c><00:19:17.760><c> it's</c><00:19:17.960><c> basically</c><00:19:18.280><c> just</c><00:19:18.400><c> your</c>\nhigh level it's basically just your\nhigh level it's basically just your validation<00:19:19.080><c> loss</c><00:19:19.919><c> uh</c><00:19:20.120><c> the</c><00:19:20.320><c> slight</c><00:19:20.679><c> difference</c>\nvalidation loss uh the slight difference\nvalidation loss uh the slight difference with<00:19:21.159><c> perplexity</c><00:19:22.000><c> is</c><00:19:22.120><c> that</c><00:19:22.280><c> we</c><00:19:22.400><c> use</c><00:19:22.600><c> something</c>\nwith perplexity is that we use something\nwith perplexity is that we use something that<00:19:23.000><c> is</c><00:19:23.120><c> slightly</c><00:19:23.440><c> more</c><00:19:23.640><c> interpretable</c>\nthat is slightly more interpretable\nthat is slightly more interpretable which<00:19:24.640><c> is</c><00:19:24.799><c> that</c><00:19:24.919><c> we</c><00:19:25.039><c> use</c><00:19:25.240><c> the</c><00:19:25.520><c> average</c><00:19:26.320><c> per</c>\nwhich is that we use the average per\nwhich is that we use the average per token<00:19:27.120><c> loss</c><00:19:27.840><c> and</c><00:19:28.000><c> then</c><00:19:28.120><c> you</c><00:19:28.320><c> expon</c><00:19:28.840><c> entiate</c><00:19:29.200><c> it</c>\ntoken loss and then you expon entiate it\ntoken loss and then you expon entiate it and<00:19:29.559><c> the</c><00:19:29.679><c> reason</c><00:19:29.880><c> why</c><00:19:30.000><c> you</c><00:19:30.120><c> exponentiate</c><00:19:30.679><c> it</c>\nand the reason why you exponentiate it\nand the reason why you exponentiate it is<00:19:31.280><c> because</c><00:19:31.520><c> you</c><00:19:31.720><c> want</c><00:19:32.240><c> I</c><00:19:32.320><c> mean</c><00:19:32.480><c> the</c><00:19:32.640><c> loss</c><00:19:33.400><c> has</c>\nis because you want I mean the loss has\nis because you want I mean the loss has a<00:19:33.720><c> log</c><00:19:34.120><c> inside</c><00:19:34.600><c> and</c><00:19:34.799><c> you</c><00:19:35.440><c> like</c><00:19:35.640><c> one</c><00:19:35.880><c> humans</c><00:19:36.159><c> are</c>\na log inside and you like one humans are\na log inside and you like one humans are actually<00:19:36.480><c> pretty</c><00:19:36.760><c> bad</c><00:19:36.880><c> at</c><00:19:37.000><c> thinking</c><00:19:37.240><c> in</c><00:19:37.360><c> log</c>\nactually pretty bad at thinking in log\nactually pretty bad at thinking in log space<00:19:38.080><c> but</c><00:19:38.200><c> two</c><00:19:38.679><c> logs</c><00:19:39.000><c> depend</c><00:19:39.280><c> on</c><00:19:39.360><c> the</c><00:19:39.520><c> base</c><00:19:39.960><c> of</c>\nspace but two logs depend on the base of\nspace but two logs depend on the base of the<00:19:40.280><c> log</c><00:19:41.120><c> uh</c><00:19:41.320><c> while</c><00:19:42.080><c> when</c><00:19:42.159><c> you</c><00:19:42.320><c> exponentiate</c>\nthe log uh while when you exponentiate\nthe log uh while when you exponentiate you<00:19:43.080><c> basically</c><00:19:43.400><c> have</c><00:19:43.600><c> everything</c><00:19:43.919><c> in</c><00:19:44.120><c> the</c><00:19:44.919><c> uh</c>\nyou basically have everything in the uh\nyou basically have everything in the uh kind<00:19:45.240><c> of</c><00:19:45.400><c> the</c><00:19:45.640><c> vocabulary</c><00:19:46.280><c> size</c><00:19:46.919><c> uh</c><00:19:47.120><c> unit</c><00:19:48.120><c> um</c>\nkind of the vocabulary size uh unit um\nkind of the vocabulary size uh unit um and<00:19:48.840><c> the</c><00:19:49.039><c> average</c><00:19:49.360><c> proten</c><00:19:49.840><c> is</c><00:19:49.960><c> just</c><00:19:50.080><c> so</c><00:19:50.280><c> that</c>\nand the average proten is just so that\nand the average proten is just so that your<00:19:50.679><c> your</c><00:19:50.840><c> complexity</c><00:19:51.360><c> is</c><00:19:51.520><c> independent</c><00:19:52.240><c> of</c>\nyour your complexity is independent of\nyour your complexity is independent of the<00:19:52.520><c> length</c><00:19:52.840><c> of</c><00:19:52.919><c> your</c><00:19:53.080><c> sequence</c><00:19:54.000><c> um</c><00:19:54.240><c> so</c>\nthe length of your sequence um so\nthe length of your sequence um so perplexity<00:19:55.080><c> is</c><00:19:55.200><c> just</c><00:19:55.360><c> two</c><00:19:55.559><c> to</c><00:19:55.679><c> the</c><00:19:55.840><c> power</c><00:19:56.760><c> uh</c>\nperplexity is just two to the power uh\nperplexity is just two to the power uh average<00:19:57.280><c> of</c><00:19:57.520><c> the</c><00:19:57.640><c> loss</c><00:19:58.000><c> of</c><00:19:58.120><c> the</c><00:19:58.280><c> sequence</c>\naverage of the loss of the sequence\naverage of the loss of the sequence um<00:20:00.159><c> so</c><00:20:00.440><c> perplexity</c><00:20:01.440><c> is</c><00:20:01.640><c> between</c><00:20:02.080><c> one</c><00:20:02.760><c> and</c><00:20:02.960><c> the</c>\num so perplexity is between one and the\num so perplexity is between one and the length<00:20:03.320><c> of</c><00:20:03.480><c> the</c><00:20:03.640><c> vocabulary</c><00:20:04.360><c> of</c><00:20:04.480><c> your</c>\nlength of the vocabulary of your\nlength of the vocabulary of your tokenizer<00:20:05.720><c> uh</c><00:20:05.840><c> one</c><00:20:06.120><c> it's</c><00:20:06.280><c> simply</c><00:20:06.880><c> well</c><00:20:07.039><c> if</c><00:20:07.159><c> you</c>\ntokenizer uh one it's simply well if you\ntokenizer uh one it's simply well if you predict<00:20:07.880><c> perfectly</c><00:20:08.280><c> the</c><00:20:08.400><c> thing</c><00:20:08.679><c> which</c><00:20:09.240><c> uh</c>\npredict perfectly the thing which uh\npredict perfectly the thing which uh every<00:20:10.080><c> word</c><00:20:10.640><c> then</c><00:20:10.919><c> every</c><00:20:11.200><c> word</c><00:20:11.559><c> will</c><00:20:11.840><c> have</c>\nevery word then every word will have\nevery word then every word will have basically<00:20:12.880><c> product</c><00:20:13.320><c> of</c><00:20:13.559><c> ones</c><00:20:14.480><c> uh</c><00:20:14.600><c> so</c><00:20:14.840><c> the</c><00:20:14.960><c> best</c>\nbasically product of ones uh so the best\nbasically product of ones uh so the best perplexity<00:20:15.679><c> you</c><00:20:15.720><c> can</c><00:20:15.840><c> have</c><00:20:15.960><c> is</c><00:20:16.159><c> one</c><00:20:16.760><c> if</c><00:20:16.919><c> you</c>\nperplexity you can have is one if you\nperplexity you can have is one if you really<00:20:17.360><c> have</c><00:20:17.559><c> no</c><00:20:17.799><c> idea</c><00:20:18.120><c> you</c><00:20:18.320><c> basically</c>\nreally have no idea you basically\nreally have no idea you basically predict<00:20:19.080><c> with</c><00:20:19.280><c> one</c><00:20:19.559><c> divided</c><00:20:19.960><c> by</c><00:20:20.559><c> uh</c><00:20:20.720><c> size</c><00:20:20.960><c> of</c>\npredict with one divided by uh size of\npredict with one divided by uh size of vocabulary<00:20:22.240><c> um</c><00:20:22.480><c> and</c><00:20:22.600><c> then</c><00:20:22.720><c> you</c><00:20:22.840><c> do</c><00:20:22.960><c> simple</c>\nvocabulary um and then you do simple\nvocabulary um and then you do simple math<00:20:23.480><c> and</c><00:20:23.559><c> you</c><00:20:23.679><c> basically</c><00:20:24.000><c> get</c><00:20:24.200><c> perplexity</c><00:20:25.080><c> of</c>\nmath and you basically get perplexity of\nmath and you basically get perplexity of size<00:20:25.520><c> of</c><00:20:25.720><c> vocabulary</c><00:20:26.720><c> uh</c><00:20:26.799><c> so</c><00:20:26.960><c> the</c><00:20:27.080><c> intuition</c>\nsize of vocabulary uh so the intuition\nsize of vocabulary uh so the intuition of<00:20:27.720><c> perplexity</c><00:20:28.280><c> is</c><00:20:28.400><c> that</c><00:20:28.760><c> basically</c><00:20:29.120><c> the</c>\nof perplexity is that basically the\nof perplexity is that basically the number<00:20:29.520><c> of</c><00:20:29.720><c> tokens</c><00:20:30.120><c> that</c><00:20:30.240><c> your</c><00:20:30.400><c> model</c><00:20:30.720><c> is</c><00:20:30.880><c> kind</c>\nnumber of tokens that your model is kind\nnumber of tokens that your model is kind of<00:20:31.240><c> hesitating</c><00:20:31.799><c> between</c><00:20:32.760><c> uh</c><00:20:32.919><c> so</c><00:20:33.080><c> if</c><00:20:33.240><c> you</c><00:20:33.440><c> if</c>\nof hesitating between uh so if you if\nof hesitating between uh so if you if your<00:20:33.640><c> model</c><00:20:33.880><c> is</c><00:20:34.039><c> perfect</c><00:20:34.559><c> it</c><00:20:34.679><c> doesn't</c>\nyour model is perfect it doesn't\nyour model is perfect it doesn't hesitate<00:20:35.440><c> it</c><00:20:35.600><c> know</c><00:20:35.799><c> exactly</c><00:20:36.080><c> the</c><00:20:36.240><c> word</c><00:20:36.640><c> if</c><00:20:36.840><c> it</c>\nhesitate it know exactly the word if it\nhesitate it know exactly the word if it really<00:20:37.440><c> has</c><00:20:37.640><c> no</c><00:20:37.840><c> idea</c><00:20:38.360><c> then</c><00:20:38.480><c> it</c><00:20:38.679><c> hesitates</c>\nreally has no idea then it hesitates\nreally has no idea then it hesitates between<00:20:40.240><c> uh</c><00:20:40.760><c> all</c><00:20:40.960><c> of</c><00:20:41.120><c> the</c>\nbetween uh all of the\nbetween uh all of the vocabulary<00:20:43.600><c> uh</c><00:20:43.880><c> so</c><00:20:44.200><c> perplexity</c><00:20:45.200><c> really</c>\nvocabulary uh so perplexity really\nvocabulary uh so perplexity really improved<00:20:46.200><c> that's</c><00:20:46.520><c> perplexity</c><00:20:47.400><c> on</c><00:20:47.559><c> a</c><00:20:47.720><c> standard</c>\nimproved that's perplexity on a standard\nimproved that's perplexity on a standard data<00:20:48.360><c> set</c><00:20:48.520><c> between</c><00:20:48.799><c> 2017</c><00:20:49.440><c> and</c><00:20:49.840><c> 2023</c><00:20:50.840><c> it</c><00:20:51.000><c> it</c>\ndata set between 2017 and 2023 it it\ndata set between 2017 and 2023 it it went<00:20:51.400><c> from</c><00:20:51.760><c> kind</c><00:20:51.919><c> of</c><00:20:52.080><c> 70</c><00:20:52.720><c> tokens</c><00:20:53.360><c> to</c><00:20:53.640><c> less</c><00:20:53.799><c> than</c>\nwent from kind of 70 tokens to less than\nwent from kind of 70 tokens to less than 10<00:20:54.240><c> tokens</c><00:20:55.000><c> over</c><00:20:55.280><c> these</c><00:20:55.520><c> five</c><00:20:55.720><c> six</c><00:20:56.000><c> years</c><00:20:56.520><c> so</c>\n10 tokens over these five six years so\n10 tokens over these five six years so that<00:20:56.799><c> means</c><00:20:57.000><c> that</c><00:20:57.120><c> the</c><00:20:57.240><c> models</c><00:20:57.880><c> were</c>\nthat means that the models were\nthat means that the models were previously<00:20:58.720><c> as</c><00:20:58.880><c> dating</c><00:20:59.159><c> between</c><00:20:59.480><c> 70</c><00:21:00.120><c> words</c>\npreviously as dating between 70 words\npreviously as dating between 70 words every<00:21:00.720><c> time</c><00:21:01.159><c> it</c><00:21:01.280><c> was</c><00:21:01.480><c> generating</c><00:21:01.880><c> a</c><00:21:02.039><c> word</c><00:21:02.440><c> and</c>\nevery time it was generating a word and\nevery time it was generating a word and now<00:21:02.720><c> it's</c><00:21:03.000><c> as</c><00:21:03.159><c> dating</c><00:21:03.440><c> between</c><00:21:03.720><c> like</c><00:21:03.880><c> less</c>\nnow it's as dating between like less\nnow it's as dating between like less than<00:21:04.200><c> 10</c><00:21:04.440><c> words</c><00:21:05.120><c> so</c><00:21:05.240><c> that's</c><00:21:05.440><c> much</c><00:21:05.799><c> better</c>\nthan 10 words so that's much better\nthan 10 words so that's much better perplexity<00:21:07.320><c> is</c><00:21:07.480><c> actually</c><00:21:07.720><c> not</c><00:21:07.919><c> used</c><00:21:08.320><c> anymore</c>\nperplexity is actually not used anymore\nperplexity is actually not used anymore in<00:21:08.919><c> academic</c><00:21:09.360><c> benchmarking</c><00:21:10.200><c> mostly</c><00:21:10.559><c> because</c>\nin academic benchmarking mostly because\nin academic benchmarking mostly because it<00:21:10.799><c> depends</c><00:21:11.080><c> on</c><00:21:11.159><c> the</c><00:21:11.279><c> tokenizers</c><00:21:11.880><c> that</c><00:21:11.960><c> you</c>\nit depends on the tokenizers that you\nit depends on the tokenizers that you use<00:21:12.880><c> uh</c><00:21:12.960><c> it</c><00:21:13.120><c> depends</c><00:21:13.400><c> on</c><00:21:13.640><c> the</c><00:21:13.880><c> actual</c><00:21:14.200><c> data</c>\nuse uh it depends on the actual data\nuse uh it depends on the actual data that<00:21:14.679><c> people</c><00:21:14.919><c> are</c><00:21:15.080><c> evaluating</c><00:21:15.600><c> on</c><00:21:16.200><c> but</c><00:21:16.320><c> it's</c>\nthat people are evaluating on but it's\nthat people are evaluating on but it's still<00:21:16.720><c> very</c><00:21:16.919><c> important</c><00:21:17.240><c> for</c><00:21:17.480><c> development</c><00:21:18.240><c> of</c>\nstill very important for development of\nstill very important for development of llms<00:21:19.120><c> so</c><00:21:19.360><c> when</c><00:21:19.480><c> you</c><00:21:19.760><c> when</c><00:21:19.880><c> you</c><00:21:20.000><c> actually</c><00:21:20.200><c> train</c>\nllms so when you when you actually train\nllms so when you when you actually train your<00:21:20.520><c> own</c><00:21:20.720><c> llm</c><00:21:21.360><c> people</c><00:21:21.600><c> will</c><00:21:21.840><c> still</c><00:21:22.120><c> really</c>\nyour own llm people will still really\nyour own llm people will still really look<00:21:22.880><c> at</c><00:21:23.080><c> the</c>\nlook at the\nlook at the perplexity<00:21:25.679><c> uh</c><00:21:26.240><c> one</c><00:21:26.640><c> common</c><00:21:27.120><c> other</c><00:21:27.400><c> way</c><00:21:27.760><c> and</c>\nperplexity uh one common other way and\nperplexity uh one common other way and now<00:21:28.600><c> more</c><00:21:28.720><c> common</c><00:21:29.200><c> in</c><00:21:29.480><c> Academia</c><00:21:30.200><c> of</c>\nnow more common in Academia of\nnow more common in Academia of evaluating<00:21:30.919><c> these</c><00:21:31.039><c> llms</c><00:21:31.840><c> is</c><00:21:32.000><c> just</c><00:21:32.240><c> by</c><00:21:32.760><c> taking</c>\nevaluating these llms is just by taking\nevaluating these llms is just by taking all<00:21:33.360><c> the</c><00:21:33.559><c> classical</c><00:21:34.080><c> NLP</c><00:21:34.600><c> benchmarks</c><00:21:35.120><c> and</c>\nall the classical NLP benchmarks and\nall the classical NLP benchmarks and I'll<00:21:35.440><c> give</c><00:21:35.520><c> you</c><00:21:35.640><c> a</c><00:21:35.720><c> few</c><00:21:35.919><c> examples</c><00:21:36.279><c> later</c><00:21:37.000><c> and</c>\nI'll give you a few examples later and\nI'll give you a few examples later and just<00:21:37.320><c> kind</c><00:21:37.440><c> of</c><00:21:37.600><c> aggregating</c><00:21:38.240><c> everything</c><00:21:39.200><c> um</c>\njust kind of aggregating everything um\njust kind of aggregating everything um so<00:21:39.679><c> collect</c><00:21:40.039><c> as</c><00:21:40.159><c> many</c><00:21:40.720><c> automatically</c>\nso collect as many automatically\nso collect as many automatically evaluatable<00:21:42.520><c> benchmarks</c><00:21:43.279><c> and</c><00:21:43.480><c> just</c><00:21:43.679><c> evaluate</c>\nevaluatable benchmarks and just evaluate\nevaluatable benchmarks and just evaluate across<00:21:44.440><c> all</c><00:21:44.600><c> of</c><00:21:44.799><c> them</c><00:21:45.720><c> um</c><00:21:46.600><c> so</c><00:21:47.120><c> one</c><00:21:47.799><c> such</c><00:21:48.200><c> if</c><00:21:48.520><c> uh</c>\nacross all of them um so one such if uh\nacross all of them um so one such if uh or<00:21:48.919><c> actually</c><00:21:49.240><c> two</c><00:21:49.559><c> such</c><00:21:50.279><c> uh</c><00:21:50.440><c> benchmarks</c><00:21:51.279><c> of</c>\nor actually two such uh benchmarks of\nor actually two such uh benchmarks of what<00:21:51.679><c> we</c><00:21:51.799><c> call</c><00:21:52.400><c> uh</c><00:21:52.600><c> Helm</c><00:21:53.000><c> which</c><00:21:53.080><c> is</c><00:21:53.200><c> from</c>\nwhat we call uh Helm which is from\nwhat we call uh Helm which is from Stanford<00:21:54.039><c> and</c><00:21:54.200><c> another</c><00:21:54.400><c> one</c><00:21:54.520><c> is</c><00:21:54.640><c> the</c><00:21:54.760><c> hugging</c>\nStanford and another one is the hugging\nStanford and another one is the hugging face<00:21:55.320><c> open</c><00:21:55.720><c> LM</c><00:21:56.080><c> leader</c><00:21:56.320><c> board</c><00:21:56.600><c> which</c><00:21:56.720><c> are</c><00:21:56.840><c> the</c>\nface open LM leader board which are the\nface open LM leader board which are the probably<00:21:57.440><c> two</c><00:21:57.679><c> two</c><00:21:57.880><c> most</c><00:21:58.080><c> common</c><00:21:58.320><c> ones</c><00:21:58.799><c> right</c>\nprobably two two most common ones right\nprobably two two most common ones right now<00:21:59.960><c> um</c><00:22:00.400><c> so</c><00:22:00.720><c> just</c><00:22:00.840><c> to</c><00:22:01.080><c> give</c><00:22:01.159><c> you</c><00:22:01.279><c> an</c><00:22:01.440><c> idea</c><00:22:02.039><c> in</c>\nnow um so just to give you an idea in\nnow um so just to give you an idea in Helm<00:22:02.679><c> there</c><00:22:02.799><c> are</c><00:22:03.000><c> all</c><00:22:03.120><c> of</c><00:22:03.320><c> these</c><00:22:03.520><c> type</c><00:22:03.720><c> of</c>\nHelm there are all of these type of\nHelm there are all of these type of tasks<00:22:04.720><c> which</c><00:22:04.840><c> are</c><00:22:05.120><c> mostly</c><00:22:06.080><c> things</c><00:22:06.360><c> that</c><00:22:06.559><c> can</c>\ntasks which are mostly things that can\ntasks which are mostly things that can be<00:22:06.880><c> easily</c><00:22:07.400><c> evaluated</c><00:22:08.400><c> uh</c><00:22:08.600><c> like</c><00:22:08.880><c> question</c>\nbe easily evaluated uh like question\nbe easily evaluated uh like question answering<00:22:09.799><c> so</c><00:22:10.000><c> think</c><00:22:10.200><c> about</c><00:22:10.440><c> many</c><00:22:10.720><c> different</c>\nanswering so think about many different\nanswering so think about many different question<00:22:11.400><c> answering</c><00:22:12.279><c> uh</c><00:22:12.440><c> tasks</c><00:22:13.400><c> um</c><00:22:13.679><c> and</c><00:22:13.840><c> the</c>\nquestion answering uh tasks um and the\nquestion answering uh tasks um and the benefit<00:22:14.480><c> with</c><00:22:14.640><c> question</c><00:22:14.960><c> answering</c><00:22:15.440><c> is</c><00:22:15.559><c> that</c>\nbenefit with question answering is that\nbenefit with question answering is that you<00:22:15.880><c> usually</c><00:22:16.159><c> know</c><00:22:16.400><c> what</c><00:22:16.559><c> is</c><00:22:16.679><c> the</c><00:22:16.840><c> real</c><00:22:17.200><c> answer</c>\nyou usually know what is the real answer\nyou usually know what is the real answer um<00:22:18.600><c> so</c><00:22:18.760><c> you</c><00:22:18.880><c> can</c><00:22:19.200><c> the</c><00:22:19.320><c> way</c><00:22:19.480><c> that</c><00:22:19.600><c> you</c><00:22:19.720><c> evaluate</c>\num so you can the way that you evaluate\num so you can the way that you evaluate these<00:22:20.200><c> models</c><00:22:20.480><c> and</c><00:22:20.600><c> I'll</c><00:22:20.760><c> give</c><00:22:20.840><c> you</c><00:22:20.919><c> a</c>\nthese models and I'll give you a\nthese models and I'll give you a concrete<00:22:21.400><c> example</c><00:22:21.720><c> in</c><00:22:21.880><c> one</c><00:22:22.080><c> second</c><00:22:22.960><c> um</c><00:22:23.240><c> is</c>\nconcrete example in one second um is\nconcrete example in one second um is that<00:22:23.520><c> you</c><00:22:23.600><c> can</c><00:22:23.799><c> just</c><00:22:24.039><c> look</c><00:22:24.240><c> at</c><00:22:24.760><c> How</c><00:22:25.039><c> likely</c><00:22:25.760><c> the</c>\nthat you can just look at How likely the\nthat you can just look at How likely the language<00:22:26.320><c> model</c><00:22:26.679><c> is</c><00:22:26.840><c> to</c><00:22:27.039><c> generate</c><00:22:27.640><c> the</c><00:22:27.840><c> real</c>\nlanguage model is to generate the real\nlanguage model is to generate the real answer<00:22:28.799><c> compared</c><00:22:29.159><c> to</c><00:22:29.320><c> some</c><00:22:29.600><c> other</c><00:22:29.880><c> answers</c>\nanswer compared to some other answers\nanswer compared to some other answers and<00:22:30.720><c> that's</c><00:22:30.919><c> essentially</c><00:22:31.240><c> at</c><00:22:31.320><c> a</c><00:22:31.480><c> high</c><00:22:31.640><c> level</c>\nand that's essentially at a high level\nand that's essentially at a high level how<00:22:32.200><c> you</c><00:22:32.360><c> evaluate</c><00:22:32.760><c> these</c><00:22:32.919><c> models</c><00:22:33.840><c> um</c><00:22:34.000><c> so</c><00:22:34.159><c> to</c>\nhow you evaluate these models um so to\nhow you evaluate these models um so to give<00:22:34.440><c> you</c><00:22:34.520><c> a</c><00:22:34.720><c> specific</c><00:22:35.159><c> example</c><00:22:35.679><c> mlu</c><00:22:36.279><c> is</c>\ngive you a specific example mlu is\ngive you a specific example mlu is probably<00:22:37.159><c> the</c><00:22:37.320><c> most</c><00:22:37.600><c> common</c><00:22:38.360><c> um</c><00:22:38.960><c> academic</c>\nprobably the most common um academic\nprobably the most common um academic Benchmark<00:22:40.080><c> for</c>\nBenchmark for\nBenchmark for llms<00:22:42.080><c> uh</c><00:22:42.360><c> and</c><00:22:42.960><c> this</c><00:22:43.080><c> is</c><00:22:43.240><c> just</c><00:22:43.360><c> a</c><00:22:43.520><c> collection</c><00:22:44.360><c> of</c>\nllms uh and this is just a collection of\nllms uh and this is just a collection of many<00:22:45.240><c> question</c><00:22:45.559><c> and</c><00:22:45.799><c> answers</c><00:22:46.200><c> in</c><00:22:46.320><c> all</c><00:22:46.480><c> of</c>\nmany question and answers in all of\nmany question and answers in all of those<00:22:46.880><c> domains</c><00:22:47.520><c> for</c><00:22:47.720><c> example</c><00:22:48.120><c> College</c>\nthose domains for example College\nthose domains for example College medicine<00:22:49.120><c> College</c><00:22:49.600><c> physics</c><00:22:50.320><c> astronomy</c><00:22:51.240><c> and</c>\nmedicine College physics astronomy and\nmedicine College physics astronomy and these<00:22:51.600><c> type</c><00:22:51.799><c> of</c><00:22:51.960><c> topics</c><00:22:52.640><c> and</c><00:22:52.760><c> the</c><00:22:52.919><c> questions</c>\nthese type of topics and the questions\nthese type of topics and the questions are<00:22:53.440><c> things</c><00:22:53.760><c> like</c><00:22:54.120><c> so</c><00:22:54.320><c> this</c><00:22:54.480><c> in</c><00:22:54.679><c> astronomy</c>\nare things like so this in astronomy\nare things like so this in astronomy what<00:22:55.520><c> is</c><00:22:55.880><c> true</c><00:22:56.159><c> for</c><00:22:56.559><c> type</c><00:22:56.799><c> 1</c><00:22:57.080><c> a</c><00:22:57.320><c> supernova</c><00:22:58.200><c> then</c>\nwhat is true for type 1 a supernova then\nwhat is true for type 1 a supernova then you<00:22:58.720><c> give</c><00:22:59.440><c> uh</c><00:22:59.760><c> four</c><00:23:00.240><c> different</c><00:23:00.559><c> potential</c>\nyou give uh four different potential\nyou give uh four different potential answers<00:23:01.960><c> and</c><00:23:02.080><c> you</c><00:23:02.279><c> just</c><00:23:02.600><c> ask</c><00:23:02.880><c> the</c><00:23:03.000><c> model</c><00:23:03.600><c> which</c>\nanswers and you just ask the model which\nanswers and you just ask the model which one<00:23:03.919><c> is</c><00:23:04.080><c> more</c><00:23:04.320><c> likely</c><00:23:04.720><c> so</c><00:23:05.159><c> there</c><00:23:05.240><c> are</c><00:23:05.400><c> many</c>\none is more likely so there are many\none is more likely so there are many different<00:23:05.880><c> ways</c><00:23:06.080><c> of</c><00:23:06.200><c> doing</c><00:23:06.400><c> it</c><00:23:06.760><c> either</c><00:23:06.960><c> you</c>\ndifferent ways of doing it either you\ndifferent ways of doing it either you can<00:23:07.200><c> look</c><00:23:07.320><c> at</c><00:23:07.480><c> the</c><00:23:07.600><c> likelihood</c><00:23:08.360><c> of</c><00:23:08.559><c> generating</c>\ncan look at the likelihood of generating\ncan look at the likelihood of generating all<00:23:09.320><c> these</c><00:23:09.520><c> answers</c><00:23:10.440><c> uh</c><00:23:10.559><c> or</c><00:23:10.679><c> you</c><00:23:10.799><c> can</c><00:23:10.919><c> ask</c><00:23:11.200><c> the</c>\nall these answers uh or you can ask the\nall these answers uh or you can ask the model<00:23:11.640><c> which</c><00:23:11.799><c> one</c><00:23:11.960><c> is</c><00:23:12.080><c> the</c><00:23:12.200><c> most</c><00:23:12.440><c> likely</c><00:23:13.200><c> uh</c><00:23:13.279><c> so</c>\nmodel which one is the most likely uh so\nmodel which one is the most likely uh so there<00:23:13.480><c> are</c><00:23:13.640><c> different</c><00:23:13.840><c> ways</c><00:23:14.039><c> that</c><00:23:14.120><c> you</c><00:23:14.200><c> can</c>\nthere are different ways that you can\nthere are different ways that you can promp<00:23:14.600><c> the</c><00:23:14.720><c> model</c><00:23:15.039><c> but</c><00:23:15.240><c> at</c><00:23:15.320><c> a</c><00:23:15.480><c> high</c><00:23:15.679><c> level</c><00:23:16.279><c> you</c>\npromp the model but at a high level you\npromp the model but at a high level you know<00:23:16.640><c> which</c><00:23:16.799><c> one</c><00:23:16.919><c> is</c><00:23:17.039><c> correct</c><00:23:17.559><c> and</c><00:23:17.679><c> there</c><00:23:17.760><c> are</c>\nknow which one is correct and there are\nknow which one is correct and there are three<00:23:18.120><c> other</c><00:23:18.400><c> mistakes</c><00:23:19.400><c> um</c><00:23:20.320><c> yes</c><00:23:21.320><c> kind</c>\nthree other mistakes um yes kind\nthree other mistakes um yes kind creating<00:23:22.520><c> is</c><00:23:22.679><c> like</c><00:23:22.919><c> unconstrained</c><00:23:23.720><c> text</c><00:23:24.120><c> as</c>\ncreating is like unconstrained text as\ncreating is like unconstrained text as the<00:23:24.440><c> output</c><00:23:25.039><c> yeah</c><00:23:25.640><c> how</c><00:23:25.760><c> do</c><00:23:25.880><c> you</c><00:23:26.360><c> evaluate</c><00:23:26.760><c> a</c>\nthe output yeah how do you evaluate a\nthe output yeah how do you evaluate a model<00:23:27.320><c> if</c><00:23:27.799><c> it</c><00:23:27.960><c> give</c><00:23:28.120><c> something</c><00:23:28.559><c> that's</c><00:23:29.039><c> you</c>\nmodel if it give something that's you\nmodel if it give something that's you know<00:23:29.919><c> semantically</c><00:23:30.919><c> completely</c><00:23:31.559><c> identical</c>\nknow semantically completely identical\nknow semantically completely identical but<00:23:33.080><c> is</c><00:23:33.279><c> not</c><00:23:33.559><c> the</c><00:23:33.760><c> exact</c><00:23:34.120><c> token</c><00:23:34.520><c> list</c><00:23:34.799><c> that</c>\nbut is not the exact token list that\nbut is not the exact token list that expect<00:23:35.760><c> yeah</c><00:23:35.960><c> so</c><00:23:36.120><c> that's</c><00:23:36.240><c> a</c><00:23:36.520><c> great</c><00:23:36.760><c> question</c>\nexpect yeah so that's a great question\nexpect yeah so that's a great question I'll<00:23:37.480><c> talk</c><00:23:37.679><c> more</c><00:23:37.880><c> about</c><00:23:38.159><c> that</c><00:23:38.360><c> later</c><00:23:39.000><c> here</c><00:23:39.159><c> in</c>\nI'll talk more about that later here in\nI'll talk more about that later here in this<00:23:39.520><c> case</c><00:23:39.760><c> we</c><00:23:39.919><c> don't</c><00:23:40.120><c> do</c><00:23:40.400><c> unconstrained</c><00:23:41.400><c> so</c>\nthis case we don't do unconstrained so\nthis case we don't do unconstrained so the<00:23:41.679><c> way</c><00:23:41.799><c> you</c><00:23:41.919><c> would</c><00:23:42.120><c> evaluate</c><00:23:42.640><c> MML</c><00:23:43.640><c> is</c>\nthe way you would evaluate MML is\nthe way you would evaluate MML is basically<00:23:44.279><c> either</c><00:23:44.600><c> you</c><00:23:45.320><c> you</c><00:23:45.600><c> ask</c><00:23:45.840><c> the</c><00:23:46.000><c> first</c>\nbasically either you you ask the first\nbasically either you you ask the first question<00:23:46.919><c> and</c><00:23:47.039><c> then</c><00:23:47.159><c> you</c><00:23:47.320><c> look</c><00:23:47.440><c> at</c><00:23:47.600><c> the</c>\nquestion and then you look at the\nquestion and then you look at the likelihood<00:23:48.960><c> of</c><00:23:49.120><c> the</c><00:23:49.279><c> model</c><00:23:49.600><c> generating</c><00:23:50.159><c> a</c><00:23:50.720><c> the</c>\nlikelihood of the model generating a the\nlikelihood of the model generating a the likelihood<00:23:51.320><c> of</c><00:23:51.400><c> the</c><00:23:51.520><c> model</c><00:23:51.760><c> generating</c><00:23:52.240><c> b</c><00:23:52.720><c> c</c>\nlikelihood of the model generating b c\nlikelihood of the model generating b c and<00:23:53.240><c> d</c><00:23:53.600><c> and</c><00:23:53.720><c> you</c><00:23:53.840><c> look</c><00:23:53.960><c> at</c><00:23:54.159><c> which</c><00:23:54.279><c> one</c><00:23:54.400><c> is</c><00:23:54.520><c> the</c>\nand d and you look at which one is the\nand d and you look at which one is the most<00:23:54.880><c> likely</c><00:23:55.520><c> or</c><00:23:55.799><c> you</c><00:23:55.880><c> can</c><00:23:56.080><c> as</c><00:23:56.279><c> the</c><00:23:56.440><c> model</c><00:23:57.039><c> out</c>\nmost likely or you can as the model out\nmost likely or you can as the model out of<00:23:57.400><c> ABC</c><00:23:57.960><c> d</c><00:23:58.520><c> which</c><00:23:58.640><c> one</c><00:23:58.799><c> is</c><00:23:58.919><c> the</c><00:23:59.080><c> most</c><00:23:59.320><c> likely</c>\nof ABC d which one is the most likely\nof ABC d which one is the most likely and<00:23:59.840><c> you</c><00:24:00.000><c> look</c><00:24:00.159><c> at</c><00:24:00.440><c> whe</c><00:24:00.960><c> the</c><00:24:01.080><c> to</c><00:24:01.279><c> the</c><00:24:01.400><c> most</c>\nand you look at whe the to the most\nand you look at whe the to the most likely<00:24:01.840><c> next</c><00:24:02.080><c> token</c><00:24:02.360><c> is</c><00:24:02.520><c> A</c><00:24:02.679><c> B</c><00:24:02.919><c> C</c><00:24:03.159><c> or</c><00:24:03.320><c> D</c><00:24:04.000><c> so</c><00:24:04.400><c> uh</c>\nlikely next token is A B C or D so uh\nlikely next token is A B C or D so uh you<00:24:04.559><c> can</c><00:24:04.760><c> strain</c><00:24:05.159><c> the</c><00:24:05.279><c> model</c><00:24:05.760><c> to</c><00:24:05.919><c> say</c><00:24:06.080><c> it</c><00:24:06.200><c> can</c>\nyou can strain the model to say it can\nyou can strain the model to say it can only<00:24:06.679><c> answer</c><00:24:07.039><c> these</c><00:24:07.200><c> four</c><00:24:07.919><c> things</c><00:24:08.919><c> you</c><00:24:09.039><c> say</c>\nonly answer these four things you say\nonly answer these four things you say you<00:24:09.360><c> constraint</c><00:24:09.880><c> the</c><00:24:10.039><c> model</c><00:24:10.679><c> you</c><00:24:10.880><c> mean</c><00:24:11.440><c> you</c>\nyou constraint the model you mean you\nyou constraint the model you mean you constraint<00:24:12.200><c> The</c><00:24:12.320><c> Prompt</c><00:24:12.600><c> or</c><00:24:12.720><c> do</c><00:24:12.799><c> you</c><00:24:12.919><c> mean</c><00:24:13.360><c> of</c>\nconstraint The Prompt or do you mean of\nconstraint The Prompt or do you mean of its<00:24:13.799><c> whole</c><00:24:14.080><c> probability</c><00:24:14.640><c> distribution</c>\nits whole probability distribution\nits whole probability distribution outputs<00:24:16.080><c> you</c><00:24:16.400><c> only</c><00:24:16.799><c> comparing</c><00:24:17.440><c> the</c><00:24:17.600><c> outputs</c>\noutputs you only comparing the outputs\noutputs you only comparing the outputs like<00:24:18.600><c> you're</c><00:24:18.760><c> only</c><00:24:18.960><c> comparing</c><00:24:19.360><c> the</c>\nlike you're only comparing the\nlike you're only comparing the a<00:24:21.039><c> so</c><00:24:21.440><c> uh</c><00:24:21.559><c> in</c><00:24:21.679><c> the</c><00:24:21.840><c> second</c><00:24:22.120><c> case</c><00:24:22.400><c> I</c><00:24:22.559><c> gave</c><00:24:22.720><c> you</c>\na so uh in the second case I gave you\na so uh in the second case I gave you you<00:24:23.080><c> would</c><00:24:23.240><c> do</c><00:24:23.440><c> exactly</c><00:24:23.880><c> the</c><00:24:24.200><c> I</c><00:24:24.400><c> actually</c><00:24:24.600><c> you</c>\nyou would do exactly the I actually you\nyou would do exactly the I actually you would<00:24:24.840><c> do</c><00:24:25.039><c> both</c><00:24:25.279><c> you</c><00:24:25.360><c> would</c><00:24:25.520><c> prompt</c><00:24:25.799><c> the</c><00:24:25.880><c> model</c>\nwould do both you would prompt the model\nwould do both you would prompt the model saying<00:24:26.480><c> ABC</c><00:24:26.880><c> or</c><00:24:27.039><c> D</c><00:24:27.399><c> plus</c><00:24:27.679><c> you</c><00:24:27.799><c> would</c><00:24:27.919><c> constrain</c>\nsaying ABC or D plus you would constrain\nsaying ABC or D plus you would constrain to<00:24:28.840><c> only</c><00:24:29.480><c> uh</c><00:24:29.679><c> look</c><00:24:29.919><c> at</c><00:24:30.200><c> these</c><00:24:30.399><c> two</c><00:24:30.720><c> these</c><00:24:30.919><c> four</c>\nto only uh look at these two these four\nto only uh look at these two these four tokens<00:24:32.120><c> in</c><00:24:32.240><c> the</c><00:24:32.440><c> first</c><00:24:32.720><c> case</c><00:24:32.880><c> you</c><00:24:33.000><c> don't</c><00:24:33.200><c> even</c>\ntokens in the first case you don't even\ntokens in the first case you don't even need<00:24:33.480><c> to</c><00:24:33.600><c> generate</c><00:24:34.039><c> anything</c><00:24:34.600><c> so</c><00:24:34.760><c> in</c><00:24:34.840><c> the</c>\nneed to generate anything so in the\nneed to generate anything so in the first<00:24:35.159><c> case</c><00:24:35.320><c> you</c><00:24:35.520><c> literally</c><00:24:35.919><c> just</c><00:24:36.080><c> look</c><00:24:36.559><c> given</c>\nfirst case you literally just look given\nfirst case you literally just look given that<00:24:36.919><c> it's</c><00:24:37.000><c> a</c><00:24:37.120><c> language</c><00:24:37.440><c> model</c><00:24:37.960><c> it</c><00:24:38.080><c> can</c><00:24:38.200><c> give</c><00:24:38.320><c> a</c>\nthat it's a language model it can give a\nthat it's a language model it can give a distribution<00:24:39.200><c> over</c><00:24:39.440><c> sentences</c><00:24:40.120><c> you</c><00:24:40.279><c> just</c>\ndistribution over sentences you just\ndistribution over sentences you just look<00:24:40.640><c> at</c><00:24:41.039><c> what</c><00:24:41.159><c> is</c><00:24:41.880><c> the</c><00:24:42.080><c> likelihood</c><00:24:42.520><c> of</c>\nlook at what is the likelihood of\nlook at what is the likelihood of generating<00:24:43.559><c> all</c><00:24:43.760><c> of</c><00:24:43.960><c> these</c><00:24:44.159><c> words</c><00:24:45.120><c> what</c><00:24:45.240><c> is</c>\ngenerating all of these words what is\ngenerating all of these words what is the<00:24:45.520><c> likelihood</c><00:24:45.960><c> of</c><00:24:46.080><c> generating</c><00:24:46.799><c> the</c><00:24:47.039><c> second</c>\nthe likelihood of generating the second\nthe likelihood of generating the second choice<00:24:48.320><c> and</c><00:24:48.399><c> you</c><00:24:48.600><c> just</c><00:24:48.760><c> look</c><00:24:48.960><c> at</c><00:24:49.159><c> whether</c><00:24:49.360><c> the</c>\nchoice and you just look at whether the\nchoice and you just look at whether the most<00:24:49.840><c> likely</c><00:24:50.960><c> sentence</c><00:24:51.960><c> is</c><00:24:52.200><c> actually</c><00:24:52.559><c> the</c>\nmost likely sentence is actually the\nmost likely sentence is actually the real<00:24:53.440><c> answer</c><00:24:54.440><c> so</c><00:24:54.600><c> you</c><00:24:54.679><c> don't</c><00:24:54.960><c> actually</c><00:24:55.440><c> sample</c>\nreal answer so you don't actually sample\nreal answer so you don't actually sample from<00:24:56.120><c> it</c><00:24:56.279><c> you</c><00:24:56.480><c> really</c><00:24:56.720><c> just</c><00:24:56.960><c> use</c><00:24:57.559><c> P</c><00:24:57.799><c> of</c><00:24:58.000><c> x</c><00:24:58.399><c> one</c>\nfrom it you really just use P of x one\nfrom it you really just use P of x one to<00:24:58.760><c> excel</c><00:24:59.679><c> does</c><00:24:59.799><c> that</c><00:24:59.960><c> make</c><00:25:00.360><c> sense</c><00:25:01.360><c> uh</c><00:25:01.600><c> that</c>\nto excel does that make sense uh that\nto excel does that make sense uh that being<00:25:02.000><c> said</c><00:25:02.440><c> evaluation</c><00:25:03.080><c> of</c><00:25:03.399><c> open-ended</c>\nbeing said evaluation of open-ended\nbeing said evaluation of open-ended questions<00:25:05.080><c> is</c><00:25:05.240><c> something</c><00:25:05.440><c> we're</c><00:25:05.559><c> going</c><00:25:05.640><c> to</c>\nquestions is something we're going to\nquestions is something we're going to talk<00:25:06.000><c> about</c><00:25:06.200><c> later</c><00:25:06.880><c> and</c><00:25:07.000><c> is</c><00:25:07.200><c> actually</c><00:25:07.440><c> really</c>\ntalk about later and is actually really\ntalk about later and is actually really important<00:25:08.120><c> and</c><00:25:08.279><c> really</c><00:25:08.960><c> challenging</c><00:25:09.960><c> yes</c>\nimportant and really challenging yes\nimportant and really challenging yes earlier<00:25:11.120><c> you</c><00:25:11.279><c> mentioned</c><00:25:11.640><c> that</c><00:25:12.000><c> um</c><00:25:12.240><c> like</c><00:25:13.080><c> um</c>\nearlier you mentioned that um like um\nearlier you mentioned that um like um metrics<00:25:13.679><c> like</c><00:25:14.000><c> flexity</c><00:25:14.480><c> are</c><00:25:14.640><c> not</c><00:25:15.520><c> are</c><00:25:15.679><c> not</c>\nmetrics like flexity are not are not\nmetrics like flexity are not are not like<00:25:16.240><c> usually</c><00:25:16.600><c> used</c><00:25:16.919><c> because</c><00:25:17.120><c> it</c><00:25:17.279><c> depends</c><00:25:17.600><c> on</c>\nlike usually used because it depends on\nlike usually used because it depends on like<00:25:18.200><c> how</c><00:25:18.320><c> you</c><00:25:18.440><c> do</c><00:25:18.559><c> your</c><00:25:18.720><c> terization</c><00:25:19.520><c> some</c>\nlike how you do your terization some\nlike how you do your terization some design<00:25:20.159><c> choices</c><00:25:20.919><c> I</c><00:25:21.000><c> was</c><00:25:21.120><c> wondering</c><00:25:21.480><c> if</c><00:25:21.559><c> you</c>\ndesign choices I was wondering if you\ndesign choices I was wondering if you could<00:25:21.840><c> speak</c><00:25:22.080><c> more</c><00:25:22.240><c> to</c><00:25:22.640><c> that</c><00:25:23.640><c> oh</c><00:25:24.240><c> um</c><00:25:24.760><c> yeah</c><00:25:25.080><c> so</c>\ncould speak more to that oh um yeah so\ncould speak more to that oh um yeah so think<00:25:25.799><c> about</c><00:25:26.039><c> perplexity</c><00:25:26.679><c> I</c><00:25:26.760><c> told</c><00:25:26.919><c> you</c>\nthink about perplexity I told you\nthink about perplexity I told you perplexity<00:25:27.600><c> is</c><00:25:27.720><c> between</c><00:25:28.000><c> one</c><00:25:28.600><c> and</c><00:25:28.840><c> vocabulary</c>\nperplexity is between one and vocabulary\nperplexity is between one and vocabulary size<00:25:30.159><c> so</c><00:25:30.360><c> now</c><00:25:30.559><c> imagine</c><00:25:31.039><c> that</c><00:25:31.279><c> Chad</c><00:25:31.559><c> GPT</c><00:25:32.120><c> uses</c><00:25:32.399><c> a</c>\nsize so now imagine that Chad GPT uses a\nsize so now imagine that Chad GPT uses a tokenizer<00:25:33.559><c> that</c><00:25:33.760><c> has</c><00:25:33.960><c> like</c><00:25:34.120><c> 10,000</c><00:25:34.640><c> tokens</c>\ntokenizer that has like 10,000 tokens\ntokenizer that has like 10,000 tokens but<00:25:35.799><c> Gemini</c><00:25:36.480><c> from</c><00:25:36.679><c> Google</c><00:25:36.960><c> uses</c><00:25:37.240><c> a</c><00:25:37.399><c> tokenizer</c>\nbut Gemini from Google uses a tokenizer\nbut Gemini from Google uses a tokenizer that<00:25:38.080><c> had</c><00:25:38.799><c> 100,000</c><00:25:39.799><c> uh</c><00:25:40.399><c> potential</c><00:25:40.840><c> tokens</c>\nthat had 100,000 uh potential tokens\nthat had 100,000 uh potential tokens then<00:25:41.960><c> actually</c><00:25:42.200><c> the</c><00:25:42.360><c> Gemini</c><00:25:42.840><c> one</c><00:25:43.399><c> will</c><00:25:44.039><c> will</c>\nthen actually the Gemini one will will\nthen actually the Gemini one will will have<00:25:44.919><c> like</c><00:25:45.039><c> the</c><00:25:45.240><c> upper</c><00:25:45.480><c> bound</c><00:25:45.760><c> of</c><00:25:46.000><c> the</c><00:25:46.240><c> the</c>\nhave like the upper bound of the the\nhave like the upper bound of the the perplexity<00:25:46.880><c> that</c><00:25:46.960><c> you</c><00:25:47.039><c> can</c><00:25:47.200><c> get</c><00:25:47.360><c> is</c><00:25:47.520><c> actually</c>\nperplexity that you can get is actually\nperplexity that you can get is actually worse<00:25:48.039><c> for</c><00:25:48.240><c> Gemini</c><00:25:49.200><c> than</c><00:25:49.679><c> for</c><00:25:49.919><c> Chad</c><00:25:50.200><c> GPT</c><00:25:50.840><c> does</c>\nworse for Gemini than for Chad GPT does\nworse for Gemini than for Chad GPT does that<00:25:51.159><c> make</c><00:25:51.320><c> sense</c><00:25:52.320><c> so</c><00:25:52.600><c> that's</c><00:25:52.799><c> just</c><00:25:52.919><c> an</c><00:25:53.159><c> idea</c>\nthat make sense so that's just an idea\nthat make sense so that's just an idea it's<00:25:54.320><c> actually</c><00:25:54.480><c> a</c><00:25:54.559><c> little</c><00:25:54.679><c> bit</c><00:25:54.840><c> more</c>\nit's actually a little bit more\nit's actually a little bit more complicated<00:25:55.440><c> than</c><00:25:55.559><c> that</c><00:25:55.679><c> but</c><00:25:55.799><c> that's</c><00:25:55.960><c> just</c>\ncomplicated than that but that's just\ncomplicated than that but that's just like<00:25:56.320><c> one</c><00:25:57.080><c> uh</c><00:25:57.200><c> first</c><00:25:57.480><c> or</c><00:25:57.720><c> the</c><00:25:57.840><c> bit</c><00:25:58.000><c> of</c><00:25:58.399><c> you</c><00:25:58.480><c> can</c>\nlike one uh first or the bit of you can\nlike one uh first or the bit of you can see<00:25:58.760><c> that</c><00:25:59.240><c> the</c><00:25:59.360><c> tokenizer</c><00:26:00.039><c> actually</c>\nsee that the tokenizer actually\nsee that the tokenizer actually matters<00:26:02.240><c> um</c>\nmatters um\nmatters um great<00:26:05.760><c> okay</c><00:26:06.279><c> so</c><00:26:06.559><c> evaluation</c><00:26:07.080><c> challenges</c>\ngreat okay so evaluation challenges\ngreat okay so evaluation challenges there<00:26:07.960><c> are</c><00:26:08.120><c> many</c><00:26:08.480><c> I'll</c><00:26:08.679><c> just</c><00:26:08.840><c> talk</c><00:26:09.039><c> about</c><00:26:09.279><c> two</c>\nthere are many I'll just talk about two\nthere are many I'll just talk about two really<00:26:09.799><c> briefly</c><00:26:10.760><c> uh</c><00:26:10.919><c> one</c><00:26:11.240><c> as</c><00:26:11.360><c> I</c><00:26:11.480><c> told</c><00:26:11.679><c> you</c>\nreally briefly uh one as I told you\nreally briefly uh one as I told you there<00:26:12.080><c> are</c><00:26:12.200><c> two</c><00:26:12.399><c> ways</c><00:26:12.600><c> of</c><00:26:12.720><c> doing</c><00:26:13.000><c> evaluation</c>\nthere are two ways of doing evaluation\nthere are two ways of doing evaluation for<00:26:13.919><c> these</c><00:26:14.120><c> mlu</c><00:26:14.919><c> actually</c><00:26:15.080><c> there</c><00:26:15.200><c> are</c><00:26:15.320><c> many</c>\nfor these mlu actually there are many\nfor these mlu actually there are many more<00:26:15.679><c> than</c><00:26:15.799><c> two</c><00:26:16.000><c> but</c><00:26:16.120><c> I</c><00:26:16.240><c> give</c><00:26:16.360><c> you</c><00:26:16.480><c> two</c>\nmore than two but I give you two\nmore than two but I give you two examples<00:26:17.799><c> um</c><00:26:18.240><c> and</c><00:26:18.840><c> it</c><00:26:19.000><c> happens</c><00:26:19.399><c> that</c><00:26:19.679><c> for</c><00:26:19.840><c> a</c>\nexamples um and it happens that for a\nexamples um and it happens that for a long<00:26:20.240><c> time</c><00:26:20.440><c> even</c><00:26:20.640><c> though</c><00:26:20.799><c> that</c><00:26:20.919><c> was</c><00:26:21.039><c> a</c><00:26:21.200><c> very</c>\nlong time even though that was a very\nlong time even though that was a very classical<00:26:21.840><c> Benchmark</c><00:26:22.320><c> that</c><00:26:22.480><c> everyone</c><00:26:22.720><c> used</c>\nclassical Benchmark that everyone used\nclassical Benchmark that everyone used uh<00:26:23.799><c> actually</c><00:26:24.640><c> different</c><00:26:25.640><c> uh</c><00:26:26.200><c> different</c>\nuh actually different uh different\nuh actually different uh different companies<00:26:27.120><c> and</c><00:26:27.360><c> different</c><00:26:27.919><c> um</c><00:26:28.600><c> different</c><00:26:29.279><c> uh</c>\ncompanies and different um different uh\ncompanies and different um different uh uh<00:26:30.559><c> different</c><00:26:30.840><c> organization</c><00:26:31.679><c> were</c><00:26:31.919><c> actually</c>\nuh different organization were actually\nuh different organization were actually using<00:26:32.480><c> different</c><00:26:32.799><c> ways</c><00:26:33.320><c> of</c><00:26:33.520><c> evaluating</c><00:26:34.159><c> mlu</c>\nusing different ways of evaluating mlu\nusing different ways of evaluating mlu and<00:26:35.240><c> as</c><00:26:35.320><c> a</c><00:26:35.520><c> result</c><00:26:35.840><c> you</c><00:26:36.000><c> could</c><00:26:36.200><c> you</c><00:26:36.399><c> get</c>\nand as a result you could you get\nand as a result you could you get completely<00:26:37.279><c> different</c><00:26:37.559><c> results</c><00:26:37.880><c> for</c><00:26:38.000><c> example</c>\ncompletely different results for example\ncompletely different results for example Lama\nLama\nLama 65b<00:26:40.840><c> uh</c><00:26:40.960><c> which</c><00:26:41.120><c> was</c><00:26:41.320><c> the</c><00:26:41.480><c> first</c><00:26:41.720><c> model</c><00:26:42.240><c> of</c><00:26:42.480><c> meta</c>\n65b uh which was the first model of meta\n65b uh which was the first model of meta in<00:26:42.960><c> the</c><00:26:43.080><c> Lama</c><00:26:43.480><c> series</c><00:26:44.320><c> uh</c><00:26:44.559><c> had</c><00:26:45.080><c> on</c><00:26:45.320><c> Helm</c><00:26:46.120><c> 63.7</c>\nin the Lama series uh had on Helm 63.7\nin the Lama series uh had on Helm 63.7 accuracy<00:26:47.960><c> but</c><00:26:48.120><c> on</c><00:26:48.399><c> this</c><00:26:48.679><c> other</c><00:26:49.600><c> um</c><00:26:50.159><c> Benchmark</c>\naccuracy but on this other um Benchmark\naccuracy but on this other um Benchmark had<00:26:51.080><c> like</c>\nhad like\nhad like 48.8<00:26:52.960><c> um</c><00:26:53.320><c> so</c><00:26:53.559><c> really</c><00:26:53.760><c> the</c><00:26:53.919><c> way</c><00:26:54.520><c> that</c><00:26:54.640><c> you</c>\n48.8 um so really the way that you\n48.8 um so really the way that you evaluate<00:26:55.240><c> and</c><00:26:55.320><c> this</c><00:26:55.440><c> is</c><00:26:55.559><c> not</c><00:26:55.720><c> even</c><00:26:56.080><c> talking</c>\nevaluate and this is not even talking\nevaluate and this is not even talking about<00:26:56.600><c> prompting</c><00:26:57.120><c> this</c><00:26:57.240><c> is</c><00:26:57.480><c> really</c><00:26:57.720><c> just</c><00:26:57.919><c> kind</c>\nabout prompting this is really just kind\nabout prompting this is really just kind of<00:26:58.399><c> the</c><00:26:58.559><c> the</c><00:26:58.679><c> way</c><00:26:58.880><c> that</c><00:26:59.000><c> you</c><00:26:59.200><c> evaluate</c><00:26:59.960><c> the</c><00:27:00.640><c> uh</c>\nof the the way that you evaluate the uh\nof the the way that you evaluate the uh the<00:27:00.880><c> models</c><00:27:01.240><c> prompting</c><00:27:01.640><c> is</c><00:27:01.799><c> another</c><00:27:02.120><c> issue</c><00:27:02.600><c> so</c>\nthe models prompting is another issue so\nthe models prompting is another issue so really<00:27:03.039><c> there</c><00:27:03.120><c> are</c><00:27:03.159><c> a</c><00:27:03.279><c> lot</c><00:27:03.399><c> of</c>\nreally there are a lot of\nreally there are a lot of inconsistencies<00:27:04.720><c> it's</c><00:27:04.919><c> not</c><00:27:05.159><c> as</c><00:27:05.440><c> easy</c><00:27:06.080><c> as</c><00:27:06.200><c> it</c>\ninconsistencies it's not as easy as it\ninconsistencies it's not as easy as it looks<00:27:07.240><c> uh</c><00:27:07.399><c> first</c><00:27:07.679><c> thing</c><00:27:08.159><c> yeah</c><00:27:08.320><c> sorry</c><00:27:08.960><c> how</c><00:27:09.120><c> can</c>\nlooks uh first thing yeah sorry how can\nlooks uh first thing yeah sorry how can we<00:27:09.399><c> make</c><00:27:09.520><c> sure</c><00:27:09.760><c> that</c><00:27:09.919><c> all</c><00:27:10.080><c> these</c><00:27:10.240><c> models</c><00:27:10.559><c> AR</c>\nwe make sure that all these models AR\nwe make sure that all these models AR trained<00:27:11.159><c> on</c><00:27:11.320><c> The</c><00:27:11.480><c> Benchmark</c><00:27:12.440><c> okay</c><00:27:13.440><c> second</c>\ntrained on The Benchmark okay second\ntrained on The Benchmark okay second thing<00:27:14.080><c> this</c><00:27:14.200><c> is</c><00:27:14.320><c> a</c><00:27:14.440><c> great</c><00:27:14.679><c> question</c><00:27:15.399><c> uh</c><00:27:15.520><c> chain</c>\nthing this is a great question uh chain\nthing this is a great question uh chain test<00:27:16.440><c> contamination</c><00:27:17.440><c> uh</c><00:27:17.640><c> this</c><00:27:17.760><c> is</c><00:27:18.000><c> something</c>\ntest contamination uh this is something\ntest contamination uh this is something which<00:27:19.039><c> I</c><00:27:19.120><c> would</c><00:27:19.360><c> say</c><00:27:19.760><c> is</c><00:27:20.240><c> really</c><00:27:20.679><c> important</c><00:27:21.279><c> in</c>\nwhich I would say is really important in\nwhich I would say is really important in Academia<00:27:23.200><c> in</c><00:27:23.960><c> uh</c><00:27:24.240><c> given</c><00:27:24.440><c> that</c><00:27:24.600><c> the</c><00:27:24.760><c> talk</c><00:27:24.919><c> is</c>\nAcademia in uh given that the talk is\nAcademia in uh given that the talk is mostly<00:27:25.480><c> about</c><00:27:25.679><c> training</c><00:27:26.000><c> large</c><00:27:26.320><c> language</c>\nmostly about training large language\nmostly about training large language models<00:27:27.640><c> uh</c><00:27:27.720><c> for</c><00:27:28.159><c> companies</c><00:27:28.440><c> it's</c><00:27:28.600><c> maybe</c><00:27:28.840><c> not</c>\nmodels uh for companies it's maybe not\nmodels uh for companies it's maybe not that<00:27:29.240><c> important</c><00:27:29.720><c> CU</c><00:27:29.960><c> they</c><00:27:30.159><c> know</c><00:27:30.919><c> what</c><00:27:31.120><c> they</c>\nthat important CU they know what they\nthat important CU they know what they trained<00:27:31.720><c> on</c><00:27:32.720><c> uh</c><00:27:33.320><c> for</c><00:27:33.760><c> us</c><00:27:34.279><c> we</c><00:27:34.440><c> have</c><00:27:34.600><c> no</c><00:27:34.760><c> idea</c><00:27:35.320><c> so</c>\ntrained on uh for us we have no idea so\ntrained on uh for us we have no idea so for<00:27:35.640><c> us</c><00:27:35.799><c> it's</c><00:27:35.919><c> a</c><00:27:36.080><c> real</c><00:27:36.440><c> problem</c><00:27:37.240><c> uh</c><00:27:37.360><c> so</c><00:27:37.559><c> there</c>\nfor us it's a real problem uh so there\nfor us it's a real problem uh so there are<00:27:37.840><c> many</c><00:27:38.120><c> different</c><00:27:38.480><c> ways</c><00:27:38.880><c> of</c><00:27:39.039><c> trying</c><00:27:39.320><c> to</c>\nare many different ways of trying to\nare many different ways of trying to test<00:27:40.279><c> whether</c><00:27:41.240><c> uh</c><00:27:41.519><c> the</c><00:27:41.799><c> test</c><00:27:42.120><c> set</c><00:27:42.840><c> sorry</c>\ntest whether uh the test set sorry\ntest whether uh the test set sorry whether<00:27:43.320><c> the</c><00:27:43.480><c> test</c><00:27:43.679><c> set</c><00:27:43.880><c> was</c><00:27:44.039><c> actually</c><00:27:44.320><c> in</c><00:27:44.440><c> the</c>\nwhether the test set was actually in the\nwhether the test set was actually in the training<00:27:44.840><c> Set</c><00:27:45.640><c> uh</c><00:27:45.960><c> one</c><00:27:46.559><c> kind</c><00:27:46.679><c> of</c><00:27:47.399><c> cute</c><00:27:47.760><c> trick</c>\ntraining Set uh one kind of cute trick\ntraining Set uh one kind of cute trick um<00:27:49.159><c> that</c><00:27:49.399><c> people</c><00:27:50.240><c> uh</c><00:27:50.880><c> in</c><00:27:51.080><c> in</c><00:27:51.159><c> the</c><00:27:51.320><c> lab</c><00:27:51.720><c> on</c><00:27:51.960><c> T</c><00:27:52.399><c> lab</c>\num that people uh in in the lab on T lab\num that people uh in in the lab on T lab have<00:27:52.720><c> found</c><00:27:53.120><c> is</c><00:27:53.240><c> that</c><00:27:53.399><c> what</c><00:27:53.480><c> you</c><00:27:53.559><c> can</c><00:27:53.720><c> do</c><00:27:54.320><c> is</c>\nhave found is that what you can do is\nhave found is that what you can do is that<00:27:54.799><c> given</c><00:27:55.039><c> that</c><00:27:55.279><c> most</c><00:27:55.480><c> of</c><00:27:55.640><c> the</c><00:27:55.880><c> data</c><00:27:56.200><c> set</c>\nthat given that most of the data set\nthat given that most of the data set online<00:27:56.919><c> are</c><00:27:57.120><c> not</c><00:27:57.360><c> randomized</c>\nonline are not randomized\nonline are not randomized you<00:27:58.760><c> can</c><00:27:59.000><c> just</c><00:27:59.240><c> look</c><00:27:59.720><c> at</c><00:28:00.399><c> and</c><00:28:00.519><c> in</c><00:28:00.640><c> that</c>\nyou can just look at and in that\nyou can just look at and in that language<00:28:01.080><c> models</c><00:28:01.440><c> what</c><00:28:01.519><c> they</c><00:28:01.640><c> do</c><00:28:01.760><c> is</c><00:28:01.880><c> just</c>\nlanguage models what they do is just\nlanguage models what they do is just predict<00:28:02.480><c> the</c><00:28:02.640><c> next</c><00:28:02.919><c> word</c><00:28:03.720><c> um</c><00:28:03.840><c> you</c><00:28:03.960><c> can</c><00:28:04.120><c> just</c>\npredict the next word um you can just\npredict the next word um you can just look<00:28:04.480><c> at</c><00:28:04.880><c> the</c><00:28:05.039><c> entire</c><00:28:05.519><c> test</c><00:28:05.799><c> Set</c><00:28:06.720><c> uh</c><00:28:06.919><c> what</c><00:28:07.080><c> if</c>\nlook at the entire test Set uh what if\nlook at the entire test Set uh what if you<00:28:07.440><c> generate</c><00:28:08.240><c> all</c><00:28:08.480><c> the</c><00:28:08.640><c> examples</c><00:28:09.559><c> in</c><00:28:09.799><c> order</c>\nyou generate all the examples in order\nyou generate all the examples in order versus<00:28:11.600><c> all</c><00:28:11.840><c> the</c><00:28:12.080><c> examples</c><00:28:12.640><c> in</c><00:28:12.760><c> a</c><00:28:12.919><c> different</c>\nversus all the examples in a different\nversus all the examples in a different order<00:28:13.960><c> and</c><00:28:14.320><c> if</c><00:28:14.440><c> it's</c><00:28:14.679><c> more</c><00:28:14.880><c> likely</c><00:28:15.159><c> to</c>\norder and if it's more likely to\norder and if it's more likely to generate<00:28:15.679><c> a</c><00:28:15.840><c> thing</c><00:28:16.240><c> in</c><00:28:16.440><c> order</c><00:28:17.080><c> given</c><00:28:17.320><c> that</c>\ngenerate a thing in order given that\ngenerate a thing in order given that there's<00:28:17.640><c> no</c><00:28:17.880><c> real</c><00:28:18.360><c> order</c><00:28:18.880><c> there</c><00:28:19.360><c> then</c><00:28:19.480><c> it</c>\nthere's no real order there then it\nthere's no real order there then it means<00:28:19.840><c> that</c><00:28:20.000><c> probably</c><00:28:20.279><c> was</c><00:28:20.399><c> in</c><00:28:20.480><c> a</c><00:28:20.600><c> training</c>\nmeans that probably was in a training\nmeans that probably was in a training set<00:28:21.440><c> does</c><00:28:21.559><c> that</c><00:28:21.679><c> make</c><00:28:21.919><c> sense</c><00:28:22.919><c> um</c><00:28:23.159><c> so</c><00:28:23.440><c> there</c><00:28:23.519><c> are</c>\nset does that make sense um so there are\nset does that make sense um so there are many<00:28:23.880><c> that's</c><00:28:24.039><c> like</c><00:28:24.200><c> one</c><00:28:24.320><c> of</c><00:28:24.480><c> them</c><00:28:24.760><c> there</c><00:28:24.840><c> are</c>\nmany that's like one of them there are\nmany that's like one of them there are many<00:28:25.159><c> other</c><00:28:25.320><c> ways</c><00:28:25.519><c> of</c><00:28:25.640><c> doing</c><00:28:25.880><c> it</c><00:28:26.159><c> train</c><00:28:26.480><c> test</c>\nmany other ways of doing it train test\nmany other ways of doing it train test contamination<00:28:27.880><c> again</c><00:28:28.320><c> not</c><00:28:28.519><c> that</c><00:28:28.640><c> important</c>\ncontamination again not that important\ncontamination again not that important for<00:28:29.120><c> development</c><00:28:29.720><c> really</c><00:28:29.960><c> important</c><00:28:30.279><c> for</c>\nfor development really important for\nfor development really important for academic\nacademic\nacademic benchmarking<00:28:33.279><c> great</c><00:28:33.679><c> so</c><00:28:33.840><c> there</c><00:28:33.919><c> are</c><00:28:34.039><c> many</c>\nbenchmarking great so there are many\nbenchmarking great so there are many other<00:28:34.399><c> challenges</c><00:28:34.919><c> but</c><00:28:35.200><c> uh</c><00:28:35.679><c> I'll</c><00:28:35.880><c> move</c><00:28:36.080><c> on</c><00:28:36.320><c> for</c>\nother challenges but uh I'll move on for\nother challenges but uh I'll move on for now<00:28:37.720><c> great</c><00:28:38.640><c> data</c><00:28:39.640><c> um</c><00:28:40.519><c> so</c><00:28:40.760><c> data</c><00:28:41.039><c> is</c><00:28:41.440><c> another</c>\nnow great data um so data is another\nnow great data um so data is another really<00:28:42.080><c> big</c><00:28:42.360><c> topic</c><00:28:43.120><c> um</c><00:28:43.320><c> at</c><00:28:43.440><c> a</c><00:28:43.600><c> high</c><00:28:43.799><c> level</c>\nreally big topic um at a high level\nreally big topic um at a high level people<00:28:44.720><c> just</c><00:28:44.880><c> say</c><00:28:45.200><c> oh</c><00:28:45.360><c> you</c><00:28:45.519><c> basically</c><00:28:45.840><c> train</c>\npeople just say oh you basically train\npeople just say oh you basically train large<00:28:46.559><c> language</c><00:28:46.880><c> models</c><00:28:47.240><c> on</c><00:28:47.440><c> all</c><00:28:47.640><c> of</c><00:28:47.840><c> Internet</c>\nlarge language models on all of Internet\nlarge language models on all of Internet what<00:28:48.760><c> does</c><00:28:48.919><c> that</c><00:28:49.080><c> even</c><00:28:49.279><c> mean</c><00:28:50.200><c> um</c><00:28:50.880><c> so</c><00:28:51.240><c> or</c><00:28:51.480><c> people</c>\nwhat does that even mean um so or people\nwhat does that even mean um so or people sometimes<00:28:52.080><c> say</c><00:28:52.200><c> all</c><00:28:52.360><c> of</c><00:28:52.480><c> clean</c><00:28:52.760><c> internet</c>\nsometimes say all of clean internet\nsometimes say all of clean internet which<00:28:53.519><c> is</c><00:28:53.840><c> even</c><00:28:54.080><c> less</c><00:28:54.519><c> defined</c><00:28:55.519><c> um</c><00:28:56.200><c> so</c>\nwhich is even less defined um so\nwhich is even less defined um so internet<00:28:56.880><c> is</c><00:28:57.120><c> very</c><00:28:57.360><c> dirty</c><00:28:57.840><c> and</c><00:28:58.440><c> really</c><00:28:58.679><c> not</c>\ninternet is very dirty and really not\ninternet is very dirty and really not representative<00:28:59.559><c> of</c><00:28:59.720><c> what</c><00:28:59.840><c> we</c><00:28:59.960><c> want</c><00:29:00.120><c> in</c>\nrepresentative of what we want in\nrepresentative of what we want in practice<00:29:00.919><c> if</c><00:29:01.080><c> I</c><00:29:01.279><c> download</c><00:29:02.039><c> a</c><00:29:02.240><c> random</c><00:29:02.640><c> website</c>\npractice if I download a random website\npractice if I download a random website right<00:29:03.360><c> now</c><00:29:04.120><c> you</c><00:29:04.200><c> would</c><00:29:04.360><c> be</c><00:29:04.480><c> shocked</c><00:29:04.919><c> at</c><00:29:05.120><c> what</c>\nright now you would be shocked at what\nright now you would be shocked at what is<00:29:05.559><c> in</c><00:29:05.799><c> there</c><00:29:06.039><c> it's</c><00:29:06.200><c> definitely</c><00:29:06.519><c> not</c><00:29:06.640><c> your</c>\nis in there it's definitely not your\nis in there it's definitely not your Wikipedia<00:29:08.200><c> um</c><00:29:09.080><c> so</c><00:29:10.080><c> I'll</c><00:29:10.720><c> go</c><00:29:11.000><c> really</c><00:29:11.360><c> briefly</c>\nWikipedia um so I'll go really briefly\nWikipedia um so I'll go really briefly on<00:29:12.279><c> like</c><00:29:12.440><c> what</c><00:29:12.600><c> people</c><00:29:12.880><c> do</c><00:29:13.640><c> um</c><00:29:14.200><c> I</c><00:29:14.279><c> can</c><00:29:14.440><c> answer</c>\non like what people do um I can answer\non like what people do um I can answer some<00:29:14.919><c> questions</c><00:29:15.360><c> but</c><00:29:16.000><c> I</c><00:29:16.080><c> mean</c><00:29:16.399><c> data</c><00:29:16.679><c> is</c><00:29:16.799><c> on</c><00:29:16.960><c> its</c>\nsome questions but I mean data is on its\nsome questions but I mean data is on its own<00:29:17.279><c> is</c><00:29:17.399><c> a</c><00:29:17.640><c> huge</c><00:29:18.200><c> topic</c><00:29:19.200><c> uh</c><00:29:19.399><c> basically</c><00:29:19.880><c> first</c>\nown is a huge topic uh basically first\nown is a huge topic uh basically first what<00:29:20.279><c> you</c><00:29:20.440><c> do</c><00:29:20.720><c> is</c><00:29:21.000><c> download</c><00:29:21.440><c> all</c><00:29:21.640><c> of</c><00:29:21.799><c> Internet</c>\nwhat you do is download all of Internet\nwhat you do is download all of Internet what<00:29:22.760><c> that</c><00:29:22.919><c> means</c><00:29:23.320><c> is</c><00:29:23.440><c> that</c><00:29:23.559><c> you</c><00:29:23.679><c> use</c><00:29:24.360><c> uh</c><00:29:24.679><c> web</c>\nwhat that means is that you use uh web\nwhat that means is that you use uh web crowlers<00:29:25.840><c> that</c><00:29:25.960><c> will</c><00:29:26.200><c> go</c><00:29:26.399><c> on</c><00:29:26.640><c> every</c><00:29:26.919><c> web</c><00:29:27.159><c> page</c>\ncrowlers that will go on every web page\ncrowlers that will go on every web page on<00:29:27.519><c> Internet</c><00:29:28.120><c> or</c><00:29:28.320><c> every</c><00:29:28.519><c> web</c><00:29:28.760><c> page</c><00:29:29.000><c> that</c><00:29:29.159><c> is</c><00:29:30.000><c> um</c>\non Internet or every web page that is um\non Internet or every web page that is um on<00:29:30.840><c> Google</c><00:29:31.799><c> uh</c><00:29:32.000><c> and</c><00:29:32.240><c> that</c><00:29:32.360><c> is</c><00:29:32.600><c> around</c><00:29:33.279><c> 250</c>\non Google uh and that is around 250\non Google uh and that is around 250 billion<00:29:34.600><c> pages</c><00:29:35.000><c> right</c><00:29:35.159><c> now</c><00:29:35.679><c> um</c><00:29:36.519><c> and</c><00:29:36.760><c> that's</c>\nbillion pages right now um and that's\nbillion pages right now um and that's around<00:29:37.200><c> one</c><00:29:37.440><c> petabyte</c><00:29:38.279><c> of</c><00:29:38.640><c> of</c><00:29:38.840><c> data</c><00:29:39.399><c> so</c><00:29:39.640><c> this</c>\naround one petabyte of of data so this\naround one petabyte of of data so this is<00:29:40.039><c> actually</c><00:29:40.440><c> a</c><00:29:40.640><c> common</c><00:29:41.080><c> common</c><00:29:41.399><c> C</c><00:29:41.840><c> is</c><00:29:42.000><c> one</c><00:29:42.240><c> web</c>\nis actually a common common C is one web\nis actually a common common C is one web crowler<00:29:42.880><c> so</c><00:29:43.039><c> people</c><00:29:43.240><c> will</c><00:29:43.399><c> usually</c><00:29:43.640><c> write</c>\ncrowler so people will usually write\ncrowler so people will usually write their<00:29:44.080><c> own</c><00:29:44.279><c> web</c><00:29:44.440><c> crowlers</c><00:29:45.039><c> what</c><00:29:45.159><c> they</c><00:29:45.279><c> do</c><00:29:45.440><c> is</c>\ntheir own web crowlers what they do is\ntheir own web crowlers what they do is that<00:29:45.720><c> they</c><00:29:45.840><c> use</c><00:29:46.440><c> standard</c><00:29:46.880><c> web</c><00:29:47.120><c> crowlers</c><00:29:47.600><c> and</c>\nthat they use standard web crowlers and\nthat they use standard web crowlers and we<00:29:48.039><c> common</c><00:29:48.320><c> crawl</c><00:29:48.760><c> is</c><00:29:48.919><c> one</c><00:29:49.080><c> of</c><00:29:49.240><c> them</c><00:29:49.960><c> uh</c><00:29:50.120><c> that</c>\nwe common crawl is one of them uh that\nwe common crawl is one of them uh that basically<00:29:50.679><c> every</c><00:29:50.919><c> month</c><00:29:51.559><c> adds</c><00:29:51.840><c> all</c><00:29:52.159><c> the</c><00:29:52.320><c> new</c>\nbasically every month adds all the new\nbasically every month adds all the new websites<00:29:53.519><c> that</c><00:29:53.679><c> were</c><00:29:53.919><c> added</c><00:29:54.679><c> on</c><00:29:55.159><c> uh</c><00:29:55.320><c> internet</c>\nwebsites that were added on uh internet\nwebsites that were added on uh internet that<00:29:55.799><c> are</c><00:29:55.960><c> found</c><00:29:56.279><c> by</c><00:29:56.519><c> by</c><00:29:56.679><c> Google</c><00:29:57.240><c> and</c><00:29:57.360><c> they</c><00:29:57.519><c> put</c>\nthat are found by by Google and they put\nthat are found by by Google and they put it<00:29:57.720><c> in</c><00:29:57.799><c> a</c><00:29:58.120><c> big</c><00:29:58.720><c> uh</c><00:29:58.840><c> basically</c><00:29:59.159><c> a</c><00:29:59.279><c> big</c><00:29:59.480><c> data</c><00:29:59.760><c> set</c>\nit in a big uh basically a big data set\nit in a big uh basically a big data set um<00:30:00.880><c> so</c><00:30:01.480><c> that's</c><00:30:01.640><c> on</c><00:30:01.840><c> common</c><00:30:02.120><c> call</c><00:30:02.360><c> you</c><00:30:02.440><c> have</c>\num so that's on common call you have\num so that's on common call you have around<00:30:02.799><c> 250</c><00:30:03.440><c> billion</c><00:30:03.760><c> pages</c><00:30:04.159><c> right</c><00:30:04.279><c> now</c><00:30:04.559><c> so</c><00:30:04.960><c> 1</c>\naround 250 billion pages right now so 1\naround 250 billion pages right now so 1 E6<00:30:05.799><c> gigabytes</c><00:30:06.760><c> of</c><00:30:07.120><c> data</c><00:30:08.120><c> once</c><00:30:08.279><c> you</c><00:30:08.399><c> have</c><00:30:08.640><c> this</c>\nE6 gigabytes of data once you have this\nE6 gigabytes of data once you have this uh<00:30:09.519><c> so</c><00:30:09.679><c> this</c><00:30:09.760><c> is</c><00:30:09.880><c> a</c><00:30:10.080><c> random</c><00:30:10.679><c> web</c><00:30:10.919><c> page</c><00:30:11.519><c> like</c>\nuh so this is a random web page like\nuh so this is a random web page like literally<00:30:12.120><c> random</c><00:30:13.000><c> uh</c><00:30:13.120><c> from</c><00:30:13.360><c> this</c><00:30:13.519><c> common</c>\nliterally random uh from this common\nliterally random uh from this common craw<00:30:14.519><c> and</c><00:30:14.679><c> what</c><00:30:14.799><c> you</c><00:30:14.919><c> see</c><00:30:15.159><c> is</c><00:30:15.279><c> that</c><00:30:15.519><c> one</c><00:30:15.720><c> it</c>\ncraw and what you see is that one it\ncraw and what you see is that one it really<00:30:16.039><c> doesn't</c><00:30:16.320><c> look</c><00:30:16.480><c> at</c><00:30:17.200><c> type</c><00:30:17.360><c> of</c><00:30:17.480><c> things</c>\nreally doesn't look at type of things\nreally doesn't look at type of things that<00:30:17.799><c> you</c><00:30:17.919><c> would</c><00:30:18.200><c> usually</c><00:30:18.519><c> see</c><00:30:18.840><c> but</c><00:30:19.039><c> actually</c>\nthat you would usually see but actually\nthat you would usually see but actually so<00:30:19.679><c> this</c><00:30:19.760><c> is</c><00:30:19.840><c> an</c><00:30:20.000><c> HTML</c><00:30:20.640><c> page</c><00:30:21.480><c> uh</c><00:30:21.600><c> it's</c><00:30:21.799><c> hard</c><00:30:22.080><c> to</c>\nso this is an HTML page uh it's hard to\nso this is an HTML page uh it's hard to see<00:30:22.720><c> but</c><00:30:23.200><c> if</c><00:30:23.360><c> you</c><00:30:23.679><c> look</c><00:30:24.000><c> through</c><00:30:24.720><c> you</c><00:30:24.840><c> will</c><00:30:25.039><c> see</c>\nsee but if you look through you will see\nsee but if you look through you will see some<00:30:25.799><c> content</c><00:30:26.399><c> for</c><00:30:26.679><c> example</c><00:30:27.679><c> here</c><00:30:28.039><c> here</c><00:30:29.039><c> uh</c>\nsome content for example here here uh\nsome content for example here here uh tesing<00:30:30.120><c> world</c><00:30:30.720><c> is</c><00:30:30.840><c> your</c><00:30:31.080><c> ultimate</c><00:30:31.640><c> source</c><00:30:32.039><c> for</c>\ntesing world is your ultimate source for\ntesing world is your ultimate source for the<00:30:32.440><c> system</c><00:30:32.840><c> X</c><00:30:33.200><c> high</c><00:30:33.440><c> performance</c><00:30:33.919><c> server</c><00:30:34.240><c> and</c>\nthe system X high performance server and\nthe system X high performance server and then<00:30:34.480><c> you</c><00:30:34.559><c> have</c><00:30:34.760><c> three</c><00:30:35.000><c> dots</c><00:30:35.279><c> so</c><00:30:35.399><c> you</c><00:30:35.480><c> don't</c>\nthen you have three dots so you don't\nthen you have three dots so you don't even<00:30:36.080><c> the</c><00:30:36.159><c> sentence</c><00:30:36.480><c> is</c><00:30:36.600><c> not</c><00:30:36.760><c> even</c><00:30:37.000><c> finished</c>\neven the sentence is not even finished\neven the sentence is not even finished that's<00:30:38.080><c> how</c><00:30:38.720><c> a</c><00:30:38.880><c> random</c><00:30:39.200><c> internet</c><00:30:39.559><c> looks</c><00:30:39.919><c> like</c>\nthat's how a random internet looks like\nthat's how a random internet looks like uh<00:30:41.000><c> so</c><00:30:41.159><c> of</c><00:30:41.320><c> course</c><00:30:41.600><c> it's</c><00:30:41.760><c> not</c><00:30:41.960><c> that</c><00:30:42.120><c> useful</c><00:30:42.559><c> if</c>\nuh so of course it's not that useful if\nuh so of course it's not that useful if you<00:30:42.799><c> just</c><00:30:43.000><c> train</c><00:30:43.320><c> a</c><00:30:43.519><c> like</c><00:30:43.679><c> large</c><00:30:43.960><c> language</c>\nyou just train a like large language\nyou just train a like large language model<00:30:44.440><c> to</c><00:30:44.559><c> generate</c><00:30:44.919><c> things</c><00:30:45.159><c> like</c><00:30:45.320><c> this</c><00:30:46.000><c> so</c>\nmodel to generate things like this so\nmodel to generate things like this so what<00:30:46.279><c> are</c><00:30:46.399><c> some</c><00:30:46.559><c> of</c><00:30:46.640><c> the</c><00:30:46.760><c> steps</c><00:30:46.960><c> that</c><00:30:47.039><c> are</c>\nwhat are some of the steps that are\nwhat are some of the steps that are needed<00:30:48.360><c> first</c><00:30:48.600><c> one</c><00:30:49.039><c> you</c><00:30:49.440><c> extract</c><00:30:49.840><c> the</c><00:30:50.000><c> text</c>\nneeded first one you extract the text\nneeded first one you extract the text from<00:30:50.720><c> the</c><00:30:50.840><c> HTML</c><00:30:51.360><c> so</c><00:30:51.480><c> that's</c><00:30:51.600><c> what</c><00:30:51.720><c> I</c><00:30:51.840><c> just</c><00:30:52.000><c> try</c>\nfrom the HTML so that's what I just try\nfrom the HTML so that's what I just try to<00:30:52.320><c> do</c><00:30:52.519><c> by</c><00:30:52.679><c> looking</c><00:30:53.000><c> at</c><00:30:53.519><c> uh</c><00:30:53.640><c> basically</c><00:30:53.960><c> the</c>\nto do by looking at uh basically the\nto do by looking at uh basically the correct<00:30:54.399><c> text</c><00:30:55.360><c> uh</c><00:30:55.559><c> there</c><00:30:55.679><c> are</c><00:30:55.760><c> a</c><00:30:55.880><c> lot</c><00:30:56.000><c> of</c>\ncorrect text uh there are a lot of\ncorrect text uh there are a lot of challenges<00:30:56.720><c> by</c><00:30:57.080><c> through</c><00:30:57.320><c> this</c><00:30:57.480><c> for</c><00:30:57.600><c> example</c>\nchallenges by through this for example\nchallenges by through this for example extracting<00:30:58.519><c> math</c><00:30:59.159><c> is</c><00:30:59.360><c> actually</c><00:30:59.679><c> very</c>\nextracting math is actually very\nextracting math is actually very complicated<00:31:01.080><c> but</c><00:31:01.279><c> pretty</c><00:31:01.519><c> important</c><00:31:01.799><c> for</c>\ncomplicated but pretty important for\ncomplicated but pretty important for training<00:31:02.200><c> large</c><00:31:02.440><c> language</c><00:31:02.799><c> models</c><00:31:03.679><c> um</c><00:31:03.960><c> or</c><00:31:04.240><c> for</c>\ntraining large language models um or for\ntraining large language models um or for example<00:31:04.679><c> boiler</c><00:31:05.080><c> plates</c><00:31:05.559><c> a</c><00:31:05.639><c> lot</c><00:31:05.799><c> of</c><00:31:05.960><c> your</c>\nexample boiler plates a lot of your\nexample boiler plates a lot of your forums<00:31:06.760><c> will</c><00:31:06.960><c> have</c><00:31:07.159><c> the</c><00:31:07.279><c> same</c><00:31:07.519><c> type</c><00:31:07.679><c> of</c>\nforums will have the same type of\nforums will have the same type of headers<00:31:08.240><c> the</c><00:31:08.360><c> same</c><00:31:08.639><c> type</c><00:31:08.840><c> of</c><00:31:09.159><c> Footers</c><00:31:10.080><c> uh</c><00:31:10.159><c> you</c>\nheaders the same type of Footers uh you\nheaders the same type of Footers uh you don't<00:31:10.399><c> want</c><00:31:10.519><c> to</c><00:31:10.679><c> repeat</c><00:31:11.039><c> all</c><00:31:11.120><c> of</c><00:31:11.279><c> this</c><00:31:11.399><c> in</c><00:31:11.519><c> your</c>\ndon't want to repeat all of this in your\ndon't want to repeat all of this in your data<00:31:13.440><c> um</c><00:31:14.000><c> then</c><00:31:14.159><c> you</c><00:31:14.240><c> will</c><00:31:14.480><c> filter</c><00:31:14.919><c> undesirable</c>\ndata um then you will filter undesirable\ndata um then you will filter undesirable content<00:31:16.720><c> uh</c><00:31:16.880><c> so</c><00:31:17.200><c> not</c><00:31:17.480><c> safe</c><00:31:17.760><c> for</c><00:31:18.000><c> work</c><00:31:18.559><c> harmful</c>\ncontent uh so not safe for work harmful\ncontent uh so not safe for work harmful content<00:31:19.519><c> pii</c><00:31:20.519><c> uh</c><00:31:20.600><c> so</c><00:31:20.760><c> usually</c><00:31:21.120><c> every</c><00:31:21.320><c> company</c>\ncontent pii uh so usually every company\ncontent pii uh so usually every company has<00:31:22.159><c> basically</c><00:31:22.639><c> a</c><00:31:23.120><c> a</c><00:31:23.760><c> black</c><00:31:24.200><c> list</c><00:31:24.960><c> of</c><00:31:25.200><c> websites</c>\nhas basically a a black list of websites\nhas basically a a black list of websites that<00:31:25.919><c> they</c><00:31:26.039><c> don't</c><00:31:26.200><c> want</c><00:31:26.279><c> to</c><00:31:26.399><c> train</c><00:31:26.720><c> the</c><00:31:26.840><c> models</c>\nthat they don't want to train the models\nthat they don't want to train the models on<00:31:27.600><c> that</c><00:31:27.919><c> Black</c><00:31:28.080><c> List</c><00:31:28.279><c> is</c><00:31:28.440><c> very</c><00:31:28.679><c> long</c><00:31:29.279><c> and</c><00:31:29.440><c> you</c>\non that Black List is very long and you\non that Black List is very long and you basically<00:31:29.919><c> say</c><00:31:30.120><c> if</c><00:31:30.240><c> it</c><00:31:30.320><c> comes</c><00:31:30.519><c> from</c><00:31:30.760><c> there</c><00:31:31.000><c> we</c>\nbasically say if it comes from there we\nbasically say if it comes from there we don't<00:31:31.279><c> train</c><00:31:31.519><c> on</c><00:31:31.720><c> this</c><00:31:32.080><c> there</c><00:31:32.200><c> are</c><00:31:32.360><c> other</c><00:31:32.600><c> ways</c>\ndon't train on this there are other ways\ndon't train on this there are other ways of<00:31:32.960><c> doing</c><00:31:33.399><c> these</c><00:31:33.559><c> things</c><00:31:33.919><c> is</c><00:31:34.039><c> that</c><00:31:34.159><c> you</c><00:31:34.240><c> can</c>\nof doing these things is that you can\nof doing these things is that you can train<00:31:34.639><c> a</c><00:31:34.799><c> small</c><00:31:35.279><c> model</c><00:31:35.880><c> for</c><00:31:36.039><c> classifying</c><00:31:36.600><c> what</c>\ntrain a small model for classifying what\ntrain a small model for classifying what is<00:31:36.880><c> pii</c><00:31:37.559><c> removing</c><00:31:38.120><c> these</c><00:31:38.320><c> things</c><00:31:39.240><c> um</c><00:31:39.799><c> it's</c>\nis pii removing these things um it's\nis pii removing these things um it's hard<00:31:40.440><c> every</c><00:31:40.799><c> Point</c><00:31:41.120><c> here</c><00:31:41.600><c> that</c><00:31:41.760><c> I'm</c><00:31:41.880><c> going</c><00:31:42.000><c> to</c>\nhard every Point here that I'm going to\nhard every Point here that I'm going to show<00:31:42.360><c> you</c><00:31:42.720><c> is</c><00:31:43.080><c> like</c><00:31:43.880><c> a</c><00:31:44.200><c> hard</c><00:31:44.919><c> amount</c><00:31:45.200><c> of</c><00:31:45.399><c> work</c>\nshow you is like a hard amount of work\nshow you is like a hard amount of work uh<00:31:46.399><c> but</c><00:31:46.480><c> I'm</c><00:31:46.760><c> going</c><00:31:46.880><c> to</c><00:31:47.080><c> go</c><00:31:47.279><c> go</c><00:31:47.559><c> quickly</c>\nuh but I'm going to go go quickly\nuh but I'm going to go go quickly through<00:31:48.080><c> it</c><00:31:48.279><c> so</c><00:31:48.440><c> filter</c><00:31:48.760><c> undesirable</c><00:31:49.360><c> content</c>\nthrough it so filter undesirable content\nthrough it so filter undesirable content second<00:31:50.919><c> or</c><00:31:51.240><c> fourth</c><00:31:51.960><c> is</c><00:31:52.200><c> the</c><00:31:52.360><c> dup</c><00:31:52.720><c> D</c>\nsecond or fourth is the dup D\nsecond or fourth is the dup D duplication<00:31:54.039><c> as</c><00:31:54.159><c> I</c><00:31:54.360><c> said</c><00:31:55.159><c> um</c><00:31:55.639><c> you</c><00:31:55.799><c> might</c><00:31:56.120><c> have</c>\nduplication as I said um you might have\nduplication as I said um you might have things<00:31:56.600><c> like</c><00:31:56.840><c> headers</c><00:31:57.159><c> and</c><00:31:57.279><c> Footers</c><00:31:58.039><c> in</c>\nthings like headers and Footers in\nthings like headers and Footers in forums<00:31:58.679><c> that</c><00:31:58.799><c> are</c><00:31:59.000><c> always</c><00:31:59.279><c> the</c><00:31:59.399><c> same</c><00:31:59.799><c> you</c><00:31:59.880><c> want</c>\nforums that are always the same you want\nforums that are always the same you want to<00:32:00.200><c> remove</c><00:32:00.600><c> that</c><00:32:01.159><c> another</c><00:32:01.440><c> thing</c><00:32:01.600><c> that</c><00:32:01.679><c> you</c>\nto remove that another thing that you\nto remove that another thing that you might<00:32:02.080><c> have</c><00:32:02.480><c> is</c><00:32:02.639><c> a</c><00:32:02.799><c> lot</c><00:32:02.960><c> of</c><00:32:03.159><c> URLs</c><00:32:03.919><c> that</c><00:32:04.039><c> are</c>\nmight have is a lot of URLs that are\nmight have is a lot of URLs that are different<00:32:04.840><c> but</c><00:32:05.080><c> actually</c><00:32:05.320><c> show</c><00:32:05.679><c> the</c><00:32:05.880><c> same</c>\ndifferent but actually show the same\ndifferent but actually show the same website<00:32:07.639><c> um</c><00:32:08.519><c> and</c><00:32:09.000><c> you</c><00:32:09.159><c> might</c><00:32:09.399><c> also</c><00:32:09.679><c> have</c><00:32:09.840><c> a</c><00:32:09.960><c> lot</c>\nwebsite um and you might also have a lot\nwebsite um and you might also have a lot of<00:32:10.360><c> like</c><00:32:10.720><c> U</c><00:32:11.200><c> um</c><00:32:11.679><c> paragraphs</c><00:32:12.360><c> that</c><00:32:12.519><c> come</c><00:32:12.720><c> from</c>\nof like U um paragraphs that come from\nof like U um paragraphs that come from like<00:32:13.159><c> common</c><00:32:13.480><c> books</c><00:32:14.039><c> that</c><00:32:14.159><c> are</c><00:32:14.360><c> basically</c>\nlike common books that are basically\nlike common books that are basically duplicated<00:32:15.919><c> a</c><00:32:16.080><c> thousand</c><00:32:16.399><c> times</c><00:32:16.600><c> or</c><00:32:16.760><c> 10,000</c>\nduplicated a thousand times or 10,000\nduplicated a thousand times or 10,000 times<00:32:17.440><c> on</c><00:32:17.639><c> internet</c><00:32:18.320><c> so</c><00:32:18.480><c> you</c><00:32:18.799><c> have</c><00:32:18.880><c> to</c>\ntimes on internet so you have to\ntimes on internet so you have to duplicate<00:32:20.159><c> also</c><00:32:20.480><c> very</c><00:32:20.720><c> challenging</c><00:32:21.720><c> uh</c>\nduplicate also very challenging uh\nduplicate also very challenging uh because<00:32:22.200><c> you</c><00:32:22.320><c> have</c><00:32:22.440><c> to</c><00:32:22.559><c> do</c><00:32:22.720><c> that</c><00:32:22.919><c> at</c><00:32:23.440><c> scale</c>\nbecause you have to do that at scale\nbecause you have to do that at scale once<00:32:24.600><c> you</c><00:32:24.720><c> do</c><00:32:25.000><c> duplication</c><00:32:26.000><c> you</c><00:32:26.120><c> will</c><00:32:26.279><c> do</c><00:32:26.399><c> some</c>\nonce you do duplication you will do some\nonce you do duplication you will do some heuristic<00:32:27.120><c> filtering</c><00:32:27.960><c> you</c><00:32:28.080><c> will</c><00:32:28.240><c> try</c><00:32:28.399><c> to</c>\nheuristic filtering you will try to\nheuristic filtering you will try to remove<00:32:29.399><c> low</c><00:32:29.799><c> quality</c><00:32:30.320><c> documents</c><00:32:31.320><c> uh</c><00:32:31.480><c> the</c><00:32:31.600><c> way</c>\nremove low quality documents uh the way\nremove low quality documents uh the way you<00:32:31.880><c> do</c><00:32:32.120><c> that</c><00:32:32.320><c> are</c><00:32:32.559><c> things</c><00:32:32.760><c> like</c><00:32:32.960><c> rules-based</c>\nyou do that are things like rules-based\nyou do that are things like rules-based um<00:32:34.279><c> filtering</c><00:32:35.159><c> for</c><00:32:35.360><c> example</c><00:32:35.720><c> if</c><00:32:35.880><c> you</c><00:32:36.000><c> see</c><00:32:36.200><c> that</c>\num filtering for example if you see that\num filtering for example if you see that there<00:32:36.399><c> are</c><00:32:36.519><c> some</c><00:32:36.720><c> outlier</c><00:32:37.240><c> tokens</c><00:32:37.760><c> if</c><00:32:37.840><c> the</c>\nthere are some outlier tokens if the\nthere are some outlier tokens if the distribution<00:32:38.480><c> of</c><00:32:38.600><c> tokens</c><00:32:38.919><c> in</c><00:32:39.039><c> the</c><00:32:39.200><c> website</c><00:32:39.760><c> is</c>\ndistribution of tokens in the website is\ndistribution of tokens in the website is very<00:32:40.159><c> different</c><00:32:40.440><c> than</c><00:32:40.559><c> the</c><00:32:40.679><c> usual</c>\nvery different than the usual\nvery different than the usual distribution<00:32:41.399><c> of</c><00:32:41.559><c> tokens</c><00:32:42.120><c> then</c><00:32:42.240><c> it's</c>\ndistribution of tokens then it's\ndistribution of tokens then it's probably<00:32:42.679><c> some</c><00:32:42.799><c> outlier</c><00:32:43.480><c> if</c><00:32:43.600><c> you</c><00:32:43.679><c> see</c><00:32:43.919><c> that</c>\nprobably some outlier if you see that\nprobably some outlier if you see that the<00:32:44.159><c> length</c><00:32:44.440><c> of</c><00:32:44.559><c> the</c><00:32:44.679><c> words</c><00:32:45.200><c> in</c><00:32:45.360><c> this</c><00:32:45.559><c> website</c>\nthe length of the words in this website\nthe length of the words in this website is<00:32:46.360><c> super</c><00:32:46.679><c> long</c><00:32:47.159><c> there's</c><00:32:47.399><c> something</c><00:32:47.679><c> strange</c>\nis super long there's something strange\nis super long there's something strange going<00:32:48.240><c> on</c><00:32:48.399><c> on</c><00:32:48.559><c> that</c><00:32:48.720><c> website</c><00:32:49.480><c> if</c><00:32:49.559><c> you</c><00:32:49.679><c> see</c><00:32:49.919><c> that</c>\ngoing on on that website if you see that\ngoing on on that website if you see that the<00:32:50.440><c> the</c><00:32:50.559><c> website</c><00:32:50.960><c> has</c><00:32:51.080><c> only</c><00:32:51.320><c> three</c><00:32:51.840><c> words</c>\nthe the website has only three words\nthe the website has only three words maybe<00:32:53.120><c> is</c><00:32:53.200><c> it</c><00:32:53.320><c> worth</c><00:32:53.519><c> training</c><00:32:53.799><c> on</c><00:32:53.919><c> it</c><00:32:54.080><c> maybe</c>\nmaybe is it worth training on it maybe\nmaybe is it worth training on it maybe not<00:32:54.559><c> if</c><00:32:54.679><c> it</c><00:32:54.880><c> has</c><00:32:55.159><c> like</c><00:32:55.600><c> 10</c><00:32:55.840><c> million</c><00:32:56.200><c> words</c>\nnot if it has like 10 million words\nnot if it has like 10 million words maybe<00:32:57.080><c> there's</c><00:32:57.279><c> something</c><00:32:57.519><c> also</c>\nmaybe there's something also\nmaybe there's something also wrong<00:32:58.880><c> going</c><00:32:59.080><c> on</c><00:32:59.279><c> that</c><00:32:59.440><c> page</c><00:33:00.240><c> um</c><00:33:00.440><c> so</c><00:33:00.559><c> a</c><00:33:00.639><c> lot</c><00:33:00.760><c> of</c>\nwrong going on that page um so a lot of\nwrong going on that page um so a lot of rules<00:33:01.120><c> like</c><00:33:01.320><c> this</c><00:33:01.600><c> yes</c><00:33:02.080><c> why</c><00:33:02.279><c> we</c><00:33:02.480><c> filter</c><00:33:02.919><c> out</c>\nrules like this yes why we filter out\nrules like this yes why we filter out undesirable<00:33:03.840><c> content</c><00:33:04.440><c> from</c><00:33:04.600><c> our</c><00:33:04.840><c> dat</c><00:33:05.159><c> set</c>\nundesirable content from our dat set\nundesirable content from our dat set instead<00:33:05.639><c> of</c><00:33:05.799><c> kind</c>\ninstead of kind\ninstead of kind of<00:33:07.639><c> putting</c><00:33:07.880><c> it</c><00:33:08.000><c> in</c><00:33:08.120><c> is</c><00:33:08.279><c> like</c><00:33:08.360><c> a</c><00:33:08.519><c> supervised</c>\nof putting it in is like a supervised\nof putting it in is like a supervised loss<00:33:10.200><c> right</c><00:33:10.480><c> like</c><00:33:10.720><c> can</c><00:33:10.840><c> we</c><00:33:11.039><c> not</c><00:33:11.240><c> just</c><00:33:11.360><c> say</c><00:33:11.679><c> like</c>\nloss right like can we not just say like\nloss right like can we not just say like you<00:33:12.320><c> know</c><00:33:12.559><c> here's</c><00:33:12.840><c> this</c><00:33:13.120><c> like</c><00:33:13.320><c> hate</c><00:33:13.559><c> speech</c>\nyou know here's this like hate speech\nyou know here's this like hate speech website<00:33:14.440><c> let's</c><00:33:15.200><c> actively</c><00:33:15.639><c> try</c><00:33:16.320><c> to</c><00:33:17.320><c> Let's</c>\nwebsite let's actively try to Let's\nwebsite let's actively try to Let's actively<00:33:17.919><c> penalize</c><00:33:18.320><c> the</c><00:33:18.679><c> for</c><00:33:18.960><c> generating</c>\nactively penalize the for generating\nactively penalize the for generating we'll<00:33:20.159><c> do</c><00:33:20.440><c> exactly</c><00:33:20.919><c> that</c><00:33:21.480><c> but</c><00:33:21.639><c> not</c><00:33:21.880><c> at</c><00:33:22.080><c> this</c>\nwe'll do exactly that but not at this\nwe'll do exactly that but not at this step<00:33:22.720><c> that's</c><00:33:22.960><c> where</c><00:33:23.120><c> the</c><00:33:23.440><c> posttraining</c><00:33:23.880><c> will</c>\nstep that's where the posttraining will\nstep that's where the posttraining will come<00:33:24.440><c> from</c><00:33:25.440><c> uh</c><00:33:25.799><c> pre-training</c><00:33:26.799><c> um</c><00:33:27.840><c> the</c><00:33:28.120><c> idea</c><00:33:28.559><c> is</c>\ncome from uh pre-training um the idea is\ncome from uh pre-training um the idea is just<00:33:28.880><c> to</c><00:33:29.200><c> say</c><00:33:30.200><c> I</c><00:33:30.279><c> want</c><00:33:30.399><c> to</c><00:33:30.639><c> model</c><00:33:31.279><c> kind</c><00:33:31.440><c> of</c><00:33:31.840><c> how</c>\njust to say I want to model kind of how\njust to say I want to model kind of how humans<00:33:32.440><c> speak</c><00:33:33.240><c> essentially</c><00:33:34.240><c> um</c><00:33:34.679><c> and</c><00:33:34.799><c> I</c><00:33:34.880><c> want</c>\nhumans speak essentially um and I want\nhumans speak essentially um and I want to<00:33:35.159><c> remove</c><00:33:35.480><c> all</c><00:33:35.679><c> these</c><00:33:35.880><c> like</c><00:33:36.039><c> headers</c><00:33:36.399><c> photos</c>\nto remove all these like headers photos\nto remove all these like headers photos and<00:33:37.039><c> and</c><00:33:37.200><c> menus</c><00:33:37.600><c> and</c><00:33:37.760><c> things</c><00:33:37.960><c> like</c><00:33:38.159><c> this</c><00:33:38.720><c> but</c>\nand and menus and things like this but\nand and menus and things like this but it's<00:33:38.919><c> a</c><00:33:39.080><c> very</c><00:33:39.240><c> good</c><00:33:39.960><c> uh</c><00:33:40.279><c> like</c><00:33:40.799><c> idea</c><00:33:41.120><c> that</c><00:33:41.200><c> you</c>\nit's a very good uh like idea that you\nit's a very good uh like idea that you just<00:33:41.519><c> had</c><00:33:41.679><c> and</c><00:33:41.919><c> that's</c><00:33:42.120><c> exactly</c><00:33:42.440><c> what</c><00:33:42.559><c> we'll</c>\njust had and that's exactly what we'll\njust had and that's exactly what we'll do\ndo\ndo later<00:33:45.399><c> Next</c><00:33:45.639><c> Step</c><00:33:45.960><c> modelbased</c><00:33:46.519><c> filtering</c><00:33:47.120><c> so</c>\nlater Next Step modelbased filtering so\nlater Next Step modelbased filtering so once<00:33:47.399><c> you</c><00:33:47.559><c> filtered</c><00:33:47.880><c> a</c><00:33:47.960><c> lot</c><00:33:48.120><c> of</c><00:33:48.279><c> data</c><00:33:48.840><c> what</c><00:33:48.960><c> you</c>\nonce you filtered a lot of data what you\nonce you filtered a lot of data what you will<00:33:49.240><c> do</c><00:33:49.799><c> uh</c><00:33:50.080><c> that's</c><00:33:50.279><c> actually</c><00:33:50.480><c> a</c><00:33:50.600><c> very</c><00:33:50.799><c> cute</c>\nwill do uh that's actually a very cute\nwill do uh that's actually a very cute trick<00:33:51.880><c> uh</c><00:33:52.000><c> you</c><00:33:52.080><c> will</c><00:33:52.279><c> take</c><00:33:52.480><c> all</c><00:33:52.639><c> of</c><00:33:52.919><c> Wikipedia</c>\ntrick uh you will take all of Wikipedia\ntrick uh you will take all of Wikipedia and<00:33:54.039><c> you</c><00:33:54.120><c> will</c><00:33:54.279><c> look</c><00:33:54.440><c> at</c><00:33:54.720><c> all</c><00:33:54.960><c> the</c><00:33:55.200><c> links</c><00:33:56.120><c> that</c>\nand you will look at all the links that\nand you will look at all the links that are<00:33:56.440><c> linked</c><00:33:56.760><c> through</c><00:33:57.039><c> Wikipedia</c><00:33:57.519><c> p</c>\nare linked through Wikipedia p\nare linked through Wikipedia p because<00:33:59.000><c> probably</c><00:33:59.320><c> if</c><00:33:59.440><c> something</c><00:33:59.720><c> is</c>\nbecause probably if something is\nbecause probably if something is referenced<00:34:00.240><c> by</c><00:34:00.399><c> Wikipedia</c><00:34:01.080><c> it's</c><00:34:01.279><c> probably</c>\nreferenced by Wikipedia it's probably\nreferenced by Wikipedia it's probably some<00:34:01.760><c> high</c><00:34:01.960><c> quality</c><00:34:02.320><c> website</c><00:34:03.240><c> and</c><00:34:03.480><c> you</c><00:34:03.559><c> will</c>\nsome high quality website and you will\nsome high quality website and you will train<00:34:04.080><c> a</c><00:34:04.360><c> classifier</c><00:34:05.360><c> to</c><00:34:05.639><c> predict</c><00:34:06.120><c> whether</c>\ntrain a classifier to predict whether\ntrain a classifier to predict whether something<00:34:07.240><c> comes</c><00:34:07.639><c> from</c><00:34:08.000><c> whether</c><00:34:08.240><c> a</c><00:34:08.560><c> document</c>\nsomething comes from whether a document\nsomething comes from whether a document comes<00:34:09.760><c> from</c><00:34:10.320><c> one</c><00:34:10.480><c> of</c><00:34:10.679><c> these</c><00:34:11.200><c> references</c><00:34:12.200><c> uh</c>\ncomes from one of these references uh\ncomes from one of these references uh from<00:34:12.520><c> Wikipedia</c><00:34:13.280><c> or</c><00:34:13.480><c> whether</c><00:34:13.679><c> it's</c><00:34:14.000><c> from</c><00:34:14.200><c> the</c>\nfrom Wikipedia or whether it's from the\nfrom Wikipedia or whether it's from the random<00:34:14.760><c> web</c><00:34:15.440><c> and</c><00:34:15.560><c> you</c><00:34:15.639><c> will</c><00:34:15.879><c> try</c><00:34:16.079><c> to</c><00:34:16.320><c> basically</c>\nrandom web and you will try to basically\nrandom web and you will try to basically say<00:34:17.240><c> I</c><00:34:17.320><c> want</c><00:34:17.599><c> more</c><00:34:17.919><c> of</c><00:34:18.200><c> the</c><00:34:18.760><c> things</c><00:34:19.040><c> that</c><00:34:19.240><c> come</c>\nsay I want more of the things that come\nsay I want more of the things that come from<00:34:20.040><c> Wikipedia</c><00:34:20.800><c> references</c><00:34:21.800><c> does</c><00:34:21.960><c> that</c><00:34:22.079><c> make</c>\nfrom Wikipedia references does that make\nfrom Wikipedia references does that make sense<00:34:23.760><c> so</c><00:34:23.960><c> yeah</c><00:34:24.079><c> so</c><00:34:24.200><c> you</c><00:34:24.280><c> will</c><00:34:24.440><c> train</c><00:34:24.639><c> a</c><00:34:24.960><c> a</c>\nsense so yeah so you will train a a\nsense so yeah so you will train a a machine<00:34:25.320><c> learning</c><00:34:25.960><c> uh</c><00:34:26.079><c> model</c><00:34:26.520><c> usually</c><00:34:26.960><c> also</c>\nmachine learning uh model usually also\nmachine learning uh model usually also very<00:34:27.480><c> simp</c><00:34:27.720><c> simple</c><00:34:27.960><c> models</c><00:34:28.399><c> because</c><00:34:28.520><c> you</c><00:34:28.639><c> need</c>\nvery simp simple models because you need\nvery simp simple models because you need to<00:34:28.879><c> do</c><00:34:29.079><c> that</c><00:34:29.280><c> really</c><00:34:29.520><c> at</c><00:34:29.679><c> scale</c><00:34:30.040><c> I</c><00:34:30.119><c> mean</c><00:34:30.280><c> just</c>\nto do that really at scale I mean just\nto do that really at scale I mean just think<00:34:30.599><c> about</c><00:34:30.800><c> the</c><00:34:30.919><c> 250</c><00:34:31.599><c> billion</c>\nthink about the 250 billion\nthink about the 250 billion Pages<00:34:33.960><c> uh</c><00:34:34.159><c> next</c><00:34:34.359><c> one</c><00:34:34.960><c> you</c><00:34:35.079><c> will</c><00:34:35.919><c> try</c><00:34:36.200><c> to</c>\nPages uh next one you will try to\nPages uh next one you will try to classify<00:34:36.919><c> your</c><00:34:37.119><c> data</c><00:34:37.720><c> into</c><00:34:38.079><c> different</c>\nclassify your data into different\nclassify your data into different different<00:34:39.440><c> um</c><00:34:40.280><c> domains</c><00:34:40.879><c> you</c><00:34:41.000><c> will</c><00:34:41.159><c> say</c><00:34:41.440><c> okay</c>\ndifferent um domains you will say okay\ndifferent um domains you will say okay this<00:34:41.800><c> is</c><00:34:42.359><c> entertainment</c><00:34:43.040><c> this</c><00:34:43.119><c> is</c><00:34:43.320><c> books</c><00:34:43.720><c> this</c>\nthis is entertainment this is books this\nthis is entertainment this is books this is<00:34:44.040><c> code</c><00:34:44.679><c> this</c><00:34:44.760><c> is</c><00:34:44.960><c> like</c><00:34:45.119><c> these</c><00:34:45.280><c> type</c><00:34:45.480><c> of</c>\nis code this is like these type of\nis code this is like these type of domains<00:34:46.440><c> and</c><00:34:46.599><c> then</c><00:34:46.720><c> you</c><00:34:46.839><c> will</c><00:34:47.079><c> try</c><00:34:47.320><c> to</c><00:34:48.000><c> either</c>\ndomains and then you will try to either\ndomains and then you will try to either um<00:34:49.399><c> up</c><00:34:49.760><c> or</c><00:34:50.000><c> down</c><00:34:50.359><c> weight</c><00:34:51.040><c> some</c><00:34:51.280><c> of</c><00:34:51.399><c> the</c><00:34:51.520><c> domains</c>\num up or down weight some of the domains\num up or down weight some of the domains uh<00:34:52.639><c> for</c><00:34:52.800><c> example</c><00:34:53.159><c> you</c><00:34:53.320><c> might</c><00:34:53.520><c> say</c><00:34:54.320><c> uh</c><00:34:54.399><c> you</c>\nuh for example you might say uh you\nuh for example you might say uh you might<00:34:54.720><c> see</c><00:34:54.960><c> that</c><00:34:55.159><c> actually</c><00:34:55.440><c> if</c><00:34:55.560><c> you</c><00:34:55.679><c> train</c>\nmight see that actually if you train\nmight see that actually if you train more<00:34:56.240><c> on</c><00:34:56.480><c> code</c><00:34:57.200><c> then</c><00:34:57.359><c> actually</c><00:34:57.720><c> your</c><00:34:57.839><c> model</c>\nmore on code then actually your model\nmore on code then actually your model becomes<00:34:58.400><c> bettered</c><00:34:58.640><c> on</c><00:34:58.760><c> reasoning</c><00:34:59.160><c> so</c><00:34:59.320><c> that's</c>\nbecomes bettered on reasoning so that's\nbecomes bettered on reasoning so that's something<00:34:59.760><c> that</c><00:34:59.960><c> people</c><00:35:00.720><c> usually</c><00:35:01.040><c> say</c><00:35:01.240><c> in</c><00:35:01.320><c> a</c>\nsomething that people usually say in a\nsomething that people usually say in a very<00:35:01.640><c> handwavy</c><00:35:02.200><c> way</c><00:35:02.440><c> if</c><00:35:02.560><c> you</c><00:35:02.680><c> train</c><00:35:03.200><c> your</c>\nvery handwavy way if you train your\nvery handwavy way if you train your model<00:35:03.680><c> more</c><00:35:04.000><c> code</c><00:35:04.280><c> actually</c><00:35:04.440><c> it</c><00:35:04.560><c> helps</c>\nmodel more code actually it helps\nmodel more code actually it helps reasoning<00:35:05.400><c> so</c><00:35:05.599><c> you</c><00:35:05.760><c> want</c><00:35:05.920><c> to</c><00:35:06.240><c> upweight</c><00:35:07.079><c> the</c>\nreasoning so you want to upweight the\nreasoning so you want to upweight the coding<00:35:08.079><c> uh</c><00:35:08.240><c> distribution</c><00:35:08.839><c> because</c><00:35:09.040><c> that</c>\ncoding uh distribution because that\ncoding uh distribution because that helps<00:35:09.480><c> for</c><00:35:09.720><c> General</c><00:35:10.040><c> language</c><00:35:10.400><c> modeling</c>\nhelps for General language modeling\nhelps for General language modeling skills<00:35:11.599><c> uh</c><00:35:11.720><c> books</c><00:35:12.079><c> is</c><00:35:12.200><c> usually</c><00:35:12.520><c> also</c><00:35:12.760><c> another</c>\nskills uh books is usually also another\nskills uh books is usually also another one<00:35:13.160><c> that</c><00:35:13.320><c> people</c><00:35:13.599><c> usually</c><00:35:14.320><c> um</c><00:35:15.280><c> upweight</c>\none that people usually um upweight\none that people usually um upweight entertainment<00:35:17.000><c> they</c><00:35:17.160><c> usually</c><00:35:17.720><c> downweight</c><00:35:18.720><c> uh</c>\nentertainment they usually downweight uh\nentertainment they usually downweight uh so<00:35:19.079><c> things</c><00:35:19.280><c> like</c><00:35:19.520><c> this</c><00:35:19.880><c> of</c><00:35:20.000><c> course</c><00:35:20.160><c> you</c><00:35:20.240><c> want</c>\nso things like this of course you want\nso things like this of course you want to<00:35:20.520><c> do</c><00:35:20.640><c> it</c><00:35:20.960><c> so</c><00:35:21.200><c> people</c><00:35:21.480><c> used</c><00:35:21.720><c> to</c><00:35:21.880><c> do</c><00:35:22.000><c> it</c><00:35:22.280><c> maybe</c>\nto do it so people used to do it maybe\nto do it so people used to do it maybe uh<00:35:24.160><c> kind</c><00:35:24.280><c> of</c><00:35:24.440><c> theistically</c><00:35:25.440><c> now</c><00:35:25.680><c> there's</c>\nuh kind of theistically now there's\nuh kind of theistically now there's entire<00:35:26.400><c> pipelines</c><00:35:27.040><c> that</c><00:35:27.160><c> we'll</c><00:35:27.359><c> talk</c><00:35:27.839><c> about</c>\nentire pipelines that we'll talk about\nentire pipelines that we'll talk about of<00:35:28.400><c> how</c><00:35:28.560><c> to</c><00:35:28.680><c> do</c><00:35:28.920><c> these</c><00:35:29.079><c> things</c><00:35:29.599><c> uh</c><00:35:29.800><c> slightly</c>\nof how to do these things uh slightly\nof how to do these things uh slightly more<00:35:30.800><c> um</c>\nmore um\nmore um automatically<00:35:33.680><c> and</c><00:35:33.880><c> then</c><00:35:34.200><c> at</c><00:35:34.320><c> the</c><00:35:34.480><c> end</c><00:35:34.680><c> of</c>\nautomatically and then at the end of\nautomatically and then at the end of training<00:35:35.680><c> uh</c><00:35:35.920><c> usually</c><00:35:36.680><c> train</c><00:35:37.680><c> um</c><00:35:38.079><c> after</c>\ntraining uh usually train um after\ntraining uh usually train um after training<00:35:38.640><c> on</c><00:35:38.839><c> all</c><00:35:39.000><c> of</c><00:35:39.200><c> this</c><00:35:39.480><c> data</c><00:35:39.760><c> that</c><00:35:39.880><c> we</c><00:35:40.000><c> saw</c>\ntraining on all of this data that we saw\ntraining on all of this data that we saw usually<00:35:40.839><c> train</c><00:35:41.160><c> on</c><00:35:41.400><c> very</c><00:35:41.640><c> high</c><00:35:41.839><c> quality</c><00:35:42.280><c> data</c>\nusually train on very high quality data\nusually train on very high quality data at<00:35:43.040><c> the</c><00:35:43.200><c> end</c><00:35:43.839><c> of</c><00:35:44.280><c> of</c><00:35:44.440><c> training</c><00:35:44.720><c> your</c><00:35:45.119><c> large</c>\nat the end of of training your large\nat the end of of training your large language<00:35:45.640><c> model</c><00:35:46.079><c> where</c><00:35:46.240><c> you</c><00:35:46.359><c> decrease</c><00:35:46.680><c> your</c>\nlanguage model where you decrease your\nlanguage model where you decrease your learning<00:35:47.119><c> rate</c><00:35:47.880><c> uh</c><00:35:48.040><c> and</c><00:35:48.240><c> that</c><00:35:48.520><c> basically</c>\nlearning rate uh and that basically\nlearning rate uh and that basically means<00:35:49.079><c> that</c><00:35:49.200><c> you're</c><00:35:49.440><c> kind</c><00:35:49.599><c> of</c><00:35:49.800><c> overfitting</c>\nmeans that you're kind of overfitting\nmeans that you're kind of overfitting your<00:35:50.599><c> model</c><00:35:51.160><c> on</c><00:35:51.280><c> a</c><00:35:51.480><c> very</c><00:35:51.800><c> high</c><00:35:52.000><c> quality</c><00:35:52.440><c> data</c>\nyour model on a very high quality data\nyour model on a very high quality data so<00:35:53.040><c> usually</c><00:35:53.319><c> what</c><00:35:53.440><c> you</c><00:35:53.560><c> do</c><00:35:53.839><c> there</c><00:35:54.160><c> is</c><00:35:54.280><c> like</c>\nso usually what you do there is like\nso usually what you do there is like Wikipedia<00:35:55.480><c> you</c><00:35:56.079><c> basically</c><00:35:56.640><c> overfit</c><00:35:57.079><c> on</c>\nWikipedia you basically overfit on\nWikipedia you basically overfit on Wikipedia<00:35:57.760><c> yeah</c><00:35:58.119><c> and</c><00:35:58.280><c> you</c><00:35:58.480><c> overfit</c><00:35:59.400><c> on</c><00:35:59.800><c> like</c>\nWikipedia yeah and you overfit on like\nWikipedia yeah and you overfit on like human<00:36:01.319><c> uh</c><00:36:02.040><c> data</c><00:36:02.359><c> that</c><00:36:02.480><c> was</c><00:36:03.000><c> collected</c><00:36:04.000><c> um</c><00:36:04.480><c> the</c>\nhuman uh data that was collected um the\nhuman uh data that was collected um the other<00:36:04.960><c> things</c><00:36:05.280><c> like</c><00:36:05.440><c> continual</c><00:36:05.920><c> pre-training</c>\nother things like continual pre-training\nother things like continual pre-training for<00:36:06.560><c> getting</c><00:36:06.800><c> longer</c><00:36:07.240><c> context</c><00:36:07.960><c> I'm</c><00:36:08.119><c> I'm</c><00:36:08.200><c> going</c>\nfor getting longer context I'm I'm going\nfor getting longer context I'm I'm going to<00:36:08.400><c> skip</c><00:36:08.640><c> over</c><00:36:08.880><c> all</c><00:36:09.000><c> of</c><00:36:09.160><c> these</c><00:36:09.359><c> things</c><00:36:10.079><c> uh</c><00:36:10.160><c> but</c>\nto skip over all of these things uh but\nto skip over all of these things uh but I<00:36:10.440><c> just</c><00:36:10.560><c> to</c><00:36:10.720><c> give</c><00:36:10.839><c> you</c><00:36:10.960><c> a</c><00:36:11.160><c> sense</c><00:36:11.400><c> of</c><00:36:11.599><c> how</c><00:36:11.800><c> hard</c>\nI just to give you a sense of how hard\nI just to give you a sense of how hard it<00:36:12.160><c> is</c><00:36:12.760><c> when</c><00:36:12.920><c> people</c><00:36:13.240><c> just</c><00:36:13.359><c> say</c><00:36:13.560><c> oh</c><00:36:13.680><c> I'm</c><00:36:13.760><c> going</c>\nit is when people just say oh I'm going\nit is when people just say oh I'm going to<00:36:13.960><c> train</c><00:36:14.160><c> on</c><00:36:14.440><c> internet</c><00:36:15.440><c> that's</c><00:36:15.640><c> a</c><00:36:15.760><c> lot</c><00:36:15.880><c> of</c>\nto train on internet that's a lot of\nto train on internet that's a lot of work<00:36:17.200><c> um</c><00:36:17.599><c> and</c><00:36:17.800><c> really</c><00:36:18.000><c> we</c><00:36:18.119><c> haven't</c><00:36:18.359><c> figured</c><00:36:18.599><c> it</c>\nwork um and really we haven't figured it\nwork um and really we haven't figured it out<00:36:19.160><c> yet</c><00:36:20.160><c> so</c><00:36:20.920><c> collecting</c><00:36:21.319><c> World</c><00:36:21.680><c> data</c><00:36:22.119><c> is</c><00:36:22.440><c> a</c>\nout yet so collecting World data is a\nout yet so collecting World data is a huge<00:36:23.079><c> part</c><00:36:23.319><c> of</c><00:36:23.520><c> practical</c><00:36:23.920><c> large</c><00:36:24.200><c> language</c>\nhuge part of practical large language\nhuge part of practical large language model<00:36:25.200><c> uh</c><00:36:25.319><c> some</c><00:36:25.520><c> might</c><00:36:25.640><c> say</c><00:36:25.880><c> it's</c><00:36:26.040><c> actually</c>\nmodel uh some might say it's actually\nmodel uh some might say it's actually the<00:36:26.480><c> key</c><00:36:26.920><c> yes</c>\nthe key yes\nthe key yes about<00:36:28.680><c> data</c><00:36:29.079><c> so</c><00:36:29.520><c> basic</c><00:36:29.880><c> question</c><00:36:30.119><c> so</c><00:36:30.319><c> usually</c>\nabout data so basic question so usually\nabout data so basic question so usually when<00:36:30.760><c> you</c><00:36:30.920><c> start</c><00:36:31.160><c> with</c><00:36:31.359><c> like</c><00:36:31.720><c> the</c><00:36:32.040><c> terabyte</c><00:36:33.040><c> of</c>\nwhen you start with like the terabyte of\nwhen you start with like the terabyte of data<00:36:33.800><c> after</c><00:36:34.000><c> I</c><00:36:34.160><c> go</c><00:36:34.280><c> through</c><00:36:34.480><c> all</c><00:36:34.640><c> that</c><00:36:34.839><c> steps</c>\ndata after I go through all that steps\ndata after I go through all that steps the<00:36:35.440><c> typical</c><00:36:35.839><c> amount</c><00:36:36.079><c> of</c><00:36:36.280><c> data</c><00:36:36.520><c> you</c><00:36:36.640><c> have</c><00:36:37.079><c> in</c>\nthe typical amount of data you have in\nthe typical amount of data you have in and<00:36:38.119><c> then</c><00:36:38.400><c> like</c><00:36:39.119><c> how</c><00:36:39.480><c> how</c><00:36:39.680><c> large</c><00:36:39.880><c> a</c><00:36:40.000><c> team</c><00:36:40.240><c> does</c>\nand then like how how large a team does\nand then like how how large a team does it<00:36:40.520><c> typically</c><00:36:40.960><c> think</c><00:36:41.119><c> to</c><00:36:41.319><c> go</c><00:36:41.520><c> through</c><00:36:41.800><c> all</c><00:36:41.960><c> the</c>\nit typically think to go through all the\nit typically think to go through all the steps<00:36:42.640><c> you</c><00:36:42.760><c> talk</c><00:36:43.040><c> about</c><00:36:43.599><c> so</c><00:36:43.880><c> how</c><00:36:44.280><c> is</c><00:36:44.359><c> the</c>\nsteps you talk about so how is the\nsteps you talk about so how is the question<00:36:44.800><c> how</c><00:36:45.000><c> large</c><00:36:45.200><c> is</c><00:36:45.319><c> the</c><00:36:45.520><c> data</c><00:36:45.880><c> after</c><00:36:46.119><c> you</c>\nquestion how large is the data after you\nquestion how large is the data after you filter<00:36:47.040><c> yeah</c><00:36:47.200><c> after</c><00:36:47.400><c> you</c><00:36:47.560><c> filter</c><00:36:47.960><c> and</c><00:36:48.079><c> then</c><00:36:48.200><c> to</c>\nfilter yeah after you filter and then to\nfilter yeah after you filter and then to go<00:36:48.520><c> through</c><00:36:48.680><c> all</c><00:36:48.839><c> the</c><00:36:49.000><c> step</c><00:36:49.240><c> how</c><00:36:49.400><c> large</c><00:36:49.640><c> a</c><00:36:49.800><c> team</c>\ngo through all the step how large a team\ngo through all the step how large a team do<00:36:50.200><c> you</c><00:36:50.359><c> need</c><00:36:50.640><c> to</c><00:36:50.839><c> go</c><00:36:51.000><c> through</c><00:36:51.280><c> like</c><00:36:51.560><c> the</c><00:36:52.240><c> the</c>\ndo you need to go through like the the\ndo you need to go through like the the other<00:36:52.599><c> fation</c><00:36:53.520><c> sttion</c><00:36:54.520><c> uh</c><00:36:54.960><c> how</c><00:36:55.160><c> slow</c><00:36:55.520><c> is</c><00:36:55.640><c> it</c><00:36:55.960><c> or</c>\nother fation sttion uh how slow is it or\nother fation sttion uh how slow is it or how<00:36:56.920><c> like</c><00:36:57.200><c> how</c><00:36:57.839><c> how</c><00:36:58.000><c> many</c><00:36:58.280><c> people</c><00:36:58.520><c> would</c><00:36:58.680><c> you</c>\nhow like how how many people would you\nhow like how how many people would you need<00:36:59.880><c> to</c><00:37:00.079><c> be</c><00:37:00.280><c> able</c><00:37:00.480><c> to</c><00:37:00.680><c> do</c><00:37:01.200><c> this</c><00:37:02.200><c> uh</c><00:37:02.520><c> okay</c>\nneed to be able to do this uh okay\nneed to be able to do this uh okay that's<00:37:02.800><c> a</c><00:37:02.920><c> great</c><00:37:03.160><c> question</c><00:37:03.520><c> I'm</c><00:37:03.599><c> going</c><00:37:03.720><c> to</c>\nthat's a great question I'm going to\nthat's a great question I'm going to somewhat<00:37:04.839><c> answer</c><00:37:05.240><c> about</c><00:37:05.560><c> the</c><00:37:05.720><c> data</c><00:37:06.520><c> uh</c><00:37:06.800><c> how</c>\nsomewhat answer about the data uh how\nsomewhat answer about the data uh how large<00:37:07.200><c> is</c><00:37:07.280><c> the</c><00:37:07.440><c> data</c><00:37:07.720><c> set</c><00:37:08.280><c> uh</c><00:37:08.440><c> at</c><00:37:08.520><c> the</c><00:37:08.599><c> end</c><00:37:08.760><c> of</c>\nlarge is the data set uh at the end of\nlarge is the data set uh at the end of this<00:37:09.079><c> slide</c><00:37:10.079><c> uh</c><00:37:10.560><c> for</c><00:37:11.560><c> number</c><00:37:11.800><c> of</c><00:37:12.000><c> people</c><00:37:12.240><c> that</c>\nthis slide uh for number of people that\nthis slide uh for number of people that work<00:37:12.640><c> on</c>\nwork on\nwork on it<00:37:14.520><c> um</c><00:37:14.760><c> that's</c><00:37:14.920><c> a</c><00:37:15.079><c> good</c><00:37:15.280><c> question</c><00:37:15.560><c> I'm</c>\nit um that's a good question I'm\nit um that's a good question I'm actually<00:37:16.400><c> not</c><00:37:16.680><c> quite</c><00:37:16.880><c> sure</c><00:37:17.240><c> but</c><00:37:17.359><c> I</c><00:37:17.480><c> would</c>\nactually not quite sure but I would\nactually not quite sure but I would say<00:37:19.520><c> yeah</c><00:37:19.680><c> I</c><00:37:19.800><c> actually</c><00:37:20.040><c> don't</c><00:37:20.920><c> quite</c><00:37:21.920><c> no</c><00:37:22.319><c> but</c><00:37:22.480><c> I</c>\nsay yeah I actually don't quite no but I\nsay yeah I actually don't quite no but I would<00:37:22.720><c> say</c><00:37:23.119><c> it's</c><00:37:23.480><c> probably</c><00:37:23.720><c> even</c><00:37:23.960><c> bigger</c><00:37:24.240><c> than</c>\nwould say it's probably even bigger than\nwould say it's probably even bigger than the<00:37:24.480><c> number</c><00:37:24.680><c> of</c><00:37:24.839><c> people</c><00:37:25.119><c> that</c><00:37:25.280><c> work</c><00:37:25.800><c> on</c><00:37:26.040><c> kind</c>\nthe number of people that work on kind\nthe number of people that work on kind of<00:37:26.440><c> the</c><00:37:27.200><c> two</c><00:37:27.520><c> tuning</c><00:37:27.920><c> of</c><00:37:28.040><c> the</c><00:37:28.200><c> pre-training</c><00:37:28.720><c> of</c>\nof the two tuning of the pre-training of\nof the two tuning of the pre-training of the<00:37:29.000><c> model</c><00:37:29.839><c> uh</c><00:37:29.920><c> so</c><00:37:30.079><c> the</c><00:37:30.280><c> data</c><00:37:30.560><c> is</c><00:37:30.760><c> bigger</c><00:37:31.440><c> than</c>\nthe model uh so the data is bigger than\nthe model uh so the data is bigger than kind<00:37:31.760><c> of</c><00:37:31.880><c> the</c><00:37:32.040><c> modeling</c><00:37:32.640><c> aspect</c><00:37:34.079><c> um</c><00:37:35.079><c> yeah</c><00:37:35.359><c> I</c><00:37:35.640><c> I</c>\nkind of the modeling aspect um yeah I I\nkind of the modeling aspect um yeah I I don't<00:37:36.000><c> think</c><00:37:36.160><c> I</c><00:37:36.319><c> have</c><00:37:36.440><c> a</c><00:37:36.680><c> good</c><00:37:37.560><c> sense</c><00:37:37.880><c> I</c><00:37:37.960><c> would</c>\ndon't think I have a good sense I would\ndon't think I have a good sense I would say<00:37:38.240><c> probably</c><00:37:38.440><c> in</c><00:37:38.560><c> Lama's</c><00:37:39.040><c> team</c><00:37:39.720><c> which</c><00:37:40.000><c> have</c>\nsay probably in Lama's team which have\nsay probably in Lama's team which have like<00:37:40.400><c> 70</c><00:37:40.839><c> years</c><00:37:41.160><c> people</c><00:37:41.400><c> I</c><00:37:41.440><c> would</c><00:37:41.560><c> say</c><00:37:41.720><c> maybe</c>\nlike 70 years people I would say maybe\nlike 70 years people I would say maybe 15<00:37:42.680><c> work</c><00:37:42.920><c> on</c><00:37:43.240><c> data</c><00:37:44.240><c> uh</c><00:37:45.079><c> I</c><00:37:45.760><c> yeah</c><00:37:46.760><c> all</c><00:37:46.960><c> these</c>\n15 work on data uh I yeah all these\n15 work on data uh I yeah all these things<00:37:47.240><c> you</c><00:37:47.359><c> don't</c><00:37:47.520><c> need</c><00:37:47.680><c> that</c><00:37:47.839><c> many</c><00:37:48.000><c> people</c>\nthings you don't need that many people\nthings you don't need that many people you<00:37:48.319><c> need</c><00:37:48.440><c> a</c><00:37:48.520><c> lot</c><00:37:48.680><c> of</c><00:37:48.760><c> computer</c><00:37:49.200><c> so</c><00:37:49.560><c> because</c>\nyou need a lot of computer so because\nyou need a lot of computer so because for<00:37:50.040><c> data</c><00:37:50.240><c> you</c><00:37:50.319><c> need</c><00:37:50.440><c> a</c><00:37:50.520><c> lot</c><00:37:50.599><c> of</c><00:37:51.079><c> CPUs</c><00:37:52.079><c> um</c><00:37:53.000><c> so</c>\nfor data you need a lot of CPUs um so\nfor data you need a lot of CPUs um so yeah<00:37:53.319><c> and</c><00:37:53.480><c> I'll</c><00:37:53.640><c> answer</c><00:37:53.880><c> the</c><00:37:54.119><c> second</c><00:37:54.400><c> question</c>\nyeah and I'll answer the second question\nyeah and I'll answer the second question at<00:37:55.000><c> the</c><00:37:55.119><c> end</c><00:37:55.240><c> of</c><00:37:55.359><c> this</c><00:37:55.680><c> slide</c><00:37:56.680><c> so</c><00:37:57.040><c> as</c><00:37:57.160><c> I</c><00:37:57.280><c> just</c>\nat the end of this slide so as I just\nat the end of this slide so as I just kind<00:37:57.920><c> of</c><00:37:58.480><c> alluded</c><00:37:58.920><c> to</c><00:37:59.599><c> really</c><00:37:59.839><c> we</c><00:37:59.960><c> haven't</c>\nkind of alluded to really we haven't\nkind of alluded to really we haven't solved<00:38:00.599><c> data</c><00:38:01.000><c> at</c><00:38:01.160><c> all</c><00:38:01.359><c> for</c><00:38:01.599><c> pre-training</c><00:38:02.240><c> so</c>\nsolved data at all for pre-training so\nsolved data at all for pre-training so there's<00:38:02.520><c> a</c><00:38:02.640><c> lot</c><00:38:02.760><c> of</c><00:38:02.880><c> research</c><00:38:03.280><c> that</c><00:38:03.480><c> that</c><00:38:03.599><c> has</c>\nthere's a lot of research that that has\nthere's a lot of research that that has to<00:38:03.839><c> be</c><00:38:04.000><c> done</c><00:38:04.480><c> first</c><00:38:04.760><c> how</c><00:38:04.839><c> do</c><00:38:04.960><c> you</c><00:38:05.160><c> process</c>\nto be done first how do you process\nto be done first how do you process these<00:38:05.720><c> things</c><00:38:05.920><c> super</c><00:38:06.200><c> efficiently</c><00:38:07.200><c> uh</c><00:38:07.319><c> second</c>\nthese things super efficiently uh second\nthese things super efficiently uh second how<00:38:07.760><c> do</c><00:38:07.880><c> you</c><00:38:08.000><c> balance</c><00:38:08.440><c> kind</c><00:38:08.560><c> of</c><00:38:08.760><c> like</c><00:38:09.200><c> all</c><00:38:09.319><c> of</c>\nhow do you balance kind of like all of\nhow do you balance kind of like all of these<00:38:09.640><c> different</c><00:38:09.880><c> domains</c><00:38:10.720><c> uh</c><00:38:10.839><c> can</c><00:38:10.960><c> you</c><00:38:11.079><c> do</c>\nthese different domains uh can you do\nthese different domains uh can you do synthetic<00:38:11.760><c> data</c><00:38:12.000><c> generation</c><00:38:12.480><c> that's</c>\nsynthetic data generation that's\nsynthetic data generation that's actually<00:38:12.839><c> a</c><00:38:13.000><c> big</c><00:38:13.119><c> one</c><00:38:13.319><c> right</c><00:38:13.520><c> now</c><00:38:14.319><c> uh</c><00:38:14.560><c> and</c>\nactually a big one right now uh and\nactually a big one right now uh and because<00:38:15.319><c> we</c><00:38:15.440><c> don't</c><00:38:15.760><c> have</c><00:38:16.480><c> uh</c><00:38:16.599><c> we'll</c><00:38:16.800><c> talk</c>\nbecause we don't have uh we'll talk\nbecause we don't have uh we'll talk about<00:38:17.160><c> that</c><00:38:17.280><c> later</c><00:38:17.599><c> we</c><00:38:17.720><c> don't</c><00:38:17.920><c> have</c><00:38:18.119><c> enough</c>\nabout that later we don't have enough\nabout that later we don't have enough data<00:38:18.960><c> on</c><00:38:19.119><c> the</c><00:38:19.359><c> internet</c><00:38:20.359><c> um</c><00:38:20.800><c> can</c><00:38:20.960><c> you</c><00:38:21.079><c> use</c>\ndata on the internet um can you use\ndata on the internet um can you use multimodal<00:38:22.079><c> data</c><00:38:22.440><c> instead</c><00:38:22.680><c> of</c><00:38:22.839><c> just</c><00:38:23.000><c> text</c>\nmultimodal data instead of just text\nmultimodal data instead of just text data<00:38:23.880><c> and</c><00:38:24.040><c> how</c><00:38:24.200><c> does</c><00:38:24.400><c> that</c><00:38:24.599><c> improve</c><00:38:25.200><c> even</c><00:38:25.440><c> your</c>\ndata and how does that improve even your\ndata and how does that improve even your text<00:38:26.319><c> performance</c><00:38:27.319><c> um</c>\ntext performance um\ntext performance um there's<00:38:28.520><c> a</c><00:38:28.640><c> lot</c><00:38:28.760><c> of</c><00:38:28.880><c> seccy</c><00:38:29.560><c> because</c><00:38:29.760><c> really</c>\nthere's a lot of seccy because really\nthere's a lot of seccy because really this<00:38:30.119><c> is</c><00:38:30.319><c> the</c><00:38:30.520><c> key</c><00:38:31.200><c> of</c><00:38:31.480><c> most</c><00:38:31.680><c> of</c><00:38:31.800><c> the</c><00:38:31.960><c> pre-train</c>\nthis is the key of most of the pre-train\nthis is the key of most of the pre-train pre-trained<00:38:32.920><c> large</c><00:38:33.160><c> language</c><00:38:33.440><c> models</c><00:38:34.280><c> so</c><00:38:34.440><c> for</c>\npre-trained large language models so for\npre-trained large language models so for competitive<00:38:35.119><c> Dynamics</c><00:38:36.119><c> uh</c><00:38:36.280><c> usually</c><00:38:36.720><c> these</c>\ncompetitive Dynamics uh usually these\ncompetitive Dynamics uh usually these these<00:38:37.640><c> um</c><00:38:38.480><c> these</c><00:38:39.119><c> companies</c><00:38:39.560><c> don't</c><00:38:39.839><c> talk</c>\nthese um these companies don't talk\nthese um these companies don't talk about<00:38:40.280><c> how</c><00:38:40.400><c> they</c><00:38:40.520><c> do</c><00:38:40.680><c> the</c><00:38:40.800><c> data</c><00:38:41.079><c> collection</c>\nabout how they do the data collection\nabout how they do the data collection and<00:38:42.040><c> also</c><00:38:42.240><c> there's</c><00:38:42.400><c> a</c><00:38:42.520><c> copyright</c><00:38:42.960><c> liability</c>\nand also there's a copyright liability\nand also there's a copyright liability issue<00:38:44.040><c> they</c><00:38:44.200><c> definitely</c><00:38:44.480><c> don't</c><00:38:44.560><c> want</c><00:38:44.680><c> to</c><00:38:44.800><c> tell</c>\nissue they definitely don't want to tell\nissue they definitely don't want to tell you<00:38:45.079><c> that</c><00:38:45.200><c> they've</c><00:38:45.359><c> trained</c><00:38:45.640><c> on</c><00:38:45.800><c> books</c><00:38:46.079><c> even</c>\nyou that they've trained on books even\nyou that they've trained on books even though<00:38:46.400><c> they</c><00:38:46.560><c> did</c><00:38:47.240><c> um</c><00:38:47.440><c> because</c><00:38:47.640><c> if</c><00:38:47.760><c> not</c><00:38:47.920><c> you</c>\nthough they did um because if not you\nthough they did um because if not you can<00:38:48.880><c> uh</c><00:38:48.960><c> sue</c><00:38:49.480><c> them</c><00:38:50.480><c> uh</c><00:38:50.640><c> common</c><00:38:51.000><c> academic</c>\ncan uh sue them uh common academic\ncan uh sue them uh common academic benchmarks<00:38:52.200><c> uh</c><00:38:52.319><c> so</c><00:38:52.480><c> that</c><00:38:52.599><c> will</c><00:38:52.800><c> kind</c><00:38:52.920><c> of</c>\nbenchmarks uh so that will kind of\nbenchmarks uh so that will kind of answer<00:38:53.359><c> what</c><00:38:53.480><c> you</c><00:38:53.680><c> asked</c><00:38:54.599><c> um</c><00:38:54.880><c> it</c><00:38:55.079><c> started</c><00:38:55.760><c> so</c>\nanswer what you asked um it started so\nanswer what you asked um it started so those<00:38:56.119><c> are</c><00:38:56.240><c> the</c><00:38:56.359><c> smaller</c><00:38:56.720><c> ones</c><00:38:57.520><c> it's</c><00:38:57.760><c> the</c>\nthose are the smaller ones it's the\nthose are the smaller ones it's the names<00:38:58.079><c> are</c><00:38:58.200><c> not</c><00:38:58.400><c> that</c><00:38:58.520><c> important</c><00:38:58.960><c> but</c><00:38:59.079><c> it</c>\nnames are not that important but it\nnames are not that important but it started<00:38:59.520><c> from</c><00:39:00.040><c> around</c><00:39:00.359><c> 150</c><00:39:01.079><c> billion</c><00:39:01.440><c> tokens</c>\nstarted from around 150 billion tokens\nstarted from around 150 billion tokens which<00:39:02.359><c> around</c><00:39:02.800><c> uh</c><00:39:02.920><c> 800</c><00:39:03.480><c> GB</c><00:39:03.880><c> of</c><00:39:04.040><c> data</c><00:39:04.720><c> now</c><00:39:04.920><c> it's</c>\nwhich around uh 800 GB of data now it's\nwhich around uh 800 GB of data now it's around<00:39:05.400><c> 15</c><00:39:05.760><c> trillion</c><00:39:06.160><c> of</c><00:39:06.359><c> to</c><00:39:06.640><c> 15</c><00:39:07.040><c> trillion</c>\naround 15 trillion of to 15 trillion\naround 15 trillion of to 15 trillion tokens<00:39:08.119><c> which</c><00:39:08.280><c> is</c><00:39:08.480><c> also</c><00:39:09.440><c> uh</c><00:39:09.599><c> the</c><00:39:09.839><c> size</c><00:39:10.200><c> of</c><00:39:10.599><c> the</c>\ntokens which is also uh the size of the\ntokens which is also uh the size of the models<00:39:11.200><c> that</c><00:39:11.359><c> are</c><00:39:11.880><c> right</c><00:39:12.040><c> now</c><00:39:12.200><c> the</c><00:39:12.319><c> best</c>\nmodels that are right now the best\nmodels that are right now the best models<00:39:12.760><c> are</c><00:39:12.920><c> probably</c><00:39:13.119><c> trained</c><00:39:13.400><c> on</c><00:39:13.560><c> that</c>\nmodels are probably trained on that\nmodels are probably trained on that amount<00:39:13.880><c> of</c><00:39:14.000><c> data</c><00:39:14.480><c> so</c><00:39:14.680><c> 15</c><00:39:15.000><c> trillion</c><00:39:15.400><c> tokens</c><00:39:16.400><c> uh</c>\namount of data so 15 trillion tokens uh\namount of data so 15 trillion tokens uh which<00:39:16.800><c> is</c><00:39:17.520><c> probably</c><00:39:18.520><c> I</c><00:39:18.680><c> guess</c><00:39:18.960><c> two</c><00:39:19.240><c> order</c><00:39:19.480><c> of</c>\nwhich is probably I guess two order of\nwhich is probably I guess two order of manage<00:39:19.960><c> bigger</c><00:39:20.200><c> than</c><00:39:20.359><c> that</c><00:39:20.520><c> so</c><00:39:21.280><c> 80</c><00:39:22.280><c> uh</c><00:39:22.440><c> E3</c><00:39:23.040><c> gab</c>\nmanage bigger than that so 80 uh E3 gab\nmanage bigger than that so 80 uh E3 gab so<00:39:24.119><c> that</c><00:39:24.240><c> would</c><00:39:24.440><c> be</c>\nso that would be\nso that would be around<00:39:26.680><c> 100</c><00:39:26.920><c> to</c><00:39:27.400><c> thousand</c><00:39:27.760><c> times</c><00:39:28.640><c> uh</c>\naround 100 to thousand times uh\naround 100 to thousand times uh filtering<00:39:29.359><c> of</c><00:39:29.480><c> the</c><00:39:29.599><c> common</c><00:39:29.960><c> crawl</c><00:39:30.640><c> if</c><00:39:30.760><c> I'm</c><00:39:30.920><c> not</c>\nfiltering of the common crawl if I'm not\nfiltering of the common crawl if I'm not mistaken<00:39:32.480><c> um</c><00:39:33.040><c> so</c><00:39:33.280><c> yeah</c><00:39:33.880><c> one</c><00:39:34.119><c> very</c><00:39:34.599><c> one</c><00:39:34.839><c> very</c><00:39:35.240><c> uh</c>\nmistaken um so yeah one very one very uh\nmistaken um so yeah one very one very uh famous<00:39:35.720><c> one</c><00:39:35.920><c> is</c><00:39:36.079><c> the</c><00:39:36.240><c> pile</c><00:39:37.240><c> so</c><00:39:37.440><c> this</c><00:39:37.560><c> is</c>\nfamous one is the pile so this is\nfamous one is the pile so this is academic<00:39:38.280><c> Benchmark</c><00:39:38.720><c> of</c><00:39:38.839><c> the</c><00:39:38.960><c> pile</c><00:39:39.440><c> and</c><00:39:39.560><c> we</c>\nacademic Benchmark of the pile and we\nacademic Benchmark of the pile and we can<00:39:39.880><c> just</c><00:39:40.079><c> look</c><00:39:40.280><c> at</c><00:39:40.760><c> what</c><00:39:40.920><c> distribution</c><00:39:41.400><c> of</c>\ncan just look at what distribution of\ncan just look at what distribution of data<00:39:41.839><c> they</c><00:39:42.040><c> have</c><00:39:42.640><c> it's</c><00:39:42.839><c> things</c><00:39:43.200><c> like</c><00:39:44.079><c> um</c>\ndata they have it's things like um\ndata they have it's things like um archive<00:39:45.319><c> PBM</c><00:39:45.920><c> Central</c><00:39:46.920><c> uh</c><00:39:47.079><c> which</c><00:39:47.200><c> is</c><00:39:47.319><c> all</c><00:39:47.520><c> the</c>\narchive PBM Central uh which is all the\narchive PBM Central uh which is all the the<00:39:48.280><c> biology</c><00:39:48.839><c> stuff</c><00:39:49.839><c> uh</c><00:39:50.640><c> here</c><00:39:51.119><c> it's</c><00:39:51.520><c> Wikipedia</c>\nthe biology stuff uh here it's Wikipedia\nthe biology stuff uh here it's Wikipedia you<00:39:52.280><c> see</c><00:39:52.599><c> stack</c><00:39:53.200><c> exchange</c><00:39:54.200><c> um</c><00:39:54.720><c> some</c><00:39:55.079><c> GitHub</c>\nyou see stack exchange um some GitHub\nyou see stack exchange um some GitHub and<00:39:56.280><c> some</c><00:39:56.520><c> books</c><00:39:56.880><c> and</c><00:39:57.000><c> things</c><00:39:57.200><c> like</c><00:39:57.599><c> this</c><00:39:58.280><c> um</c>\nand some books and things like this um\nand some books and things like this um again<00:39:58.760><c> this</c><00:39:58.839><c> is</c><00:39:58.960><c> on</c><00:39:59.079><c> the</c><00:39:59.200><c> smaller</c><00:39:59.599><c> side</c><00:39:59.960><c> so</c>\nagain this is on the smaller side so\nagain this is on the smaller side so this<00:40:00.359><c> is</c><00:40:00.720><c> if</c><00:40:00.800><c> we</c><00:40:00.960><c> look</c><00:40:01.160><c> at</c><00:40:01.319><c> here</c><00:40:01.480><c> this</c><00:40:01.560><c> is</c><00:40:01.640><c> on</c>\nthis is if we look at here this is on\nthis is if we look at here this is on 280b<00:40:02.760><c> so</c><00:40:02.920><c> in</c><00:40:03.079><c> reality</c><00:40:03.400><c> it's</c><00:40:03.560><c> like</c><00:40:03.839><c> 100</c><00:40:04.079><c> times</c>\n280b so in reality it's like 100 times\n280b so in reality it's like 100 times bigger<00:40:04.560><c> so</c><00:40:04.680><c> you</c><00:40:04.800><c> cannot</c><00:40:05.040><c> have</c><00:40:05.240><c> that</c><00:40:05.400><c> much</c><00:40:05.599><c> of</c>\nbigger so you cannot have that much of\nbigger so you cannot have that much of GitHub<00:40:06.240><c> and</c><00:40:06.480><c> and</c><00:40:06.720><c> of</c>\nGitHub and and of\nGitHub and and of Wikipedia<00:40:08.920><c> um</c><00:40:09.520><c> in</c><00:40:09.680><c> terms</c><00:40:09.880><c> of</c><00:40:10.000><c> close</c><00:40:10.319><c> Source</c>\nWikipedia um in terms of close Source\nWikipedia um in terms of close Source models<00:40:11.400><c> just</c><00:40:11.520><c> to</c><00:40:11.680><c> give</c><00:40:11.800><c> you</c><00:40:11.920><c> an</c><00:40:12.079><c> idea</c><00:40:12.880><c> uh</c><00:40:13.040><c> Lama</c>\nmodels just to give you an idea uh Lama\nmodels just to give you an idea uh Lama 2<00:40:14.359><c> um</c><00:40:14.680><c> it</c><00:40:14.800><c> was</c><00:40:14.960><c> trained</c><00:40:15.240><c> on</c><00:40:15.400><c> 20</c><00:40:15.760><c> two</c><00:40:16.000><c> trillion</c>\n2 um it was trained on 20 two trillion\n2 um it was trained on 20 two trillion tokens<00:40:16.920><c> lamb</c><00:40:17.319><c> 3</c><00:40:17.599><c> 15</c><00:40:17.839><c> trillion</c><00:40:18.200><c> tokens</c><00:40:18.880><c> which</c>\ntokens lamb 3 15 trillion tokens which\ntokens lamb 3 15 trillion tokens which is<00:40:19.480><c> currently</c><00:40:19.880><c> the</c><00:40:20.040><c> best</c><00:40:20.359><c> model</c><00:40:20.680><c> that</c><00:40:20.800><c> we</c><00:40:20.920><c> know</c>\nis currently the best model that we know\nis currently the best model that we know on<00:40:21.440><c> how</c><00:40:21.560><c> much</c><00:40:21.680><c> it</c><00:40:21.800><c> was</c><00:40:21.920><c> trained</c><00:40:22.200><c> on</c><00:40:22.640><c> which</c><00:40:22.760><c> is</c>\non how much it was trained on which is\non how much it was trained on which is the<00:40:23.040><c> same</c><00:40:23.240><c> thing</c><00:40:23.400><c> as</c><00:40:23.680><c> this</c><00:40:24.079><c> the</c><00:40:24.520><c> the</c><00:40:25.280><c> the</c><00:40:25.440><c> best</c>\nthe same thing as this the the the best\nthe same thing as this the the the best academic<00:40:26.319><c> or</c><00:40:26.480><c> the</c><00:40:26.640><c> biggest</c><00:40:26.960><c> academic</c>\nacademic or the biggest academic\nacademic or the biggest academic Benchmark<00:40:27.920><c> which</c><00:40:28.000><c> is</c><00:40:28.119><c> 15</c><00:40:28.359><c> trillion</c><00:40:28.720><c> tokens</c>\nBenchmark which is 15 trillion tokens\nBenchmark which is 15 trillion tokens GPD<00:40:29.760><c> 4</c><00:40:30.000><c> we</c><00:40:30.079><c> don't</c><00:40:30.280><c> really</c><00:40:30.440><c> know</c><00:40:30.640><c> but</c><00:40:30.760><c> it's</c>\nGPD 4 we don't really know but it's\nGPD 4 we don't really know but it's probably<00:40:31.119><c> in</c><00:40:31.200><c> the</c><00:40:31.280><c> same</c><00:40:31.440><c> water</c><00:40:31.680><c> of</c><00:40:31.800><c> magnitude</c>\nprobably in the same water of magnitude\nprobably in the same water of magnitude or<00:40:32.720><c> it's</c><00:40:32.880><c> probably</c><00:40:33.119><c> around</c><00:40:33.359><c> that</c><00:40:33.520><c> actually</c>\nor it's probably around that actually\nor it's probably around that actually it's<00:40:33.839><c> probably</c><00:40:34.079><c> around</c><00:40:34.280><c> 13</c><00:40:35.200><c> um</c><00:40:36.040><c> from</c><00:40:36.280><c> leaks</c><00:40:36.680><c> if</c>\nit's probably around 13 um from leaks if\nit's probably around 13 um from leaks if the<00:40:36.920><c> leaks</c><00:40:37.119><c> are</c><00:40:37.440><c> true</c>\nthe leaks are true\nthe leaks are true um<00:40:40.319><c> great</c><00:40:41.319><c> so</c><00:40:41.760><c> scaling</c><00:40:42.240><c> laws</c><00:40:43.079><c> um</c><00:40:43.440><c> any</c><00:40:43.599><c> other</c>\num great so scaling laws um any other\num great so scaling laws um any other questions<00:40:44.040><c> on</c><00:40:44.200><c> Data</c><00:40:44.480><c> before</c><00:40:44.640><c> you</c><00:40:44.760><c> go</c><00:40:44.880><c> to</c>\nquestions on Data before you go to\nquestions on Data before you go to scaling\nlaws<00:40:49.079><c> sorry</c><00:40:49.400><c> I</c><00:40:49.480><c> know</c><00:40:49.599><c> I'm</c><00:40:49.720><c> giving</c><00:40:49.920><c> you</c><00:40:50.040><c> a</c><00:40:50.160><c> lot</c>\nlaws sorry I know I'm giving you a lot\nlaws sorry I know I'm giving you a lot of<00:40:50.480><c> information</c><00:40:51.040><c> but</c><00:40:51.240><c> uh</c><00:40:51.760><c> there's</c><00:40:51.920><c> a</c><00:40:52.040><c> lot</c><00:40:52.200><c> into</c>\nof information but uh there's a lot into\nof information but uh there's a lot into training<00:40:52.800><c> at</c><00:40:52.920><c> large</c><00:40:53.200><c> language</c><00:40:53.800><c> models</c><00:40:54.800><c> great</c>\ntraining at large language models great\ntraining at large language models great scaling<00:40:56.040><c> laws</c><00:40:57.040><c> so</c><00:40:57.319><c> so</c><00:40:57.640><c> the</c><00:40:57.800><c> idea</c><00:40:58.280><c> is</c><00:40:58.400><c> that</c><00:40:58.599><c> what</c>\nscaling laws so so the idea is that what\nscaling laws so so the idea is that what people<00:40:58.960><c> saw</c><00:40:59.839><c> um</c><00:41:00.200><c> around</c><00:41:00.520><c> 2020</c><00:41:01.280><c> or</c><00:41:01.440><c> at</c><00:41:01.520><c> least</c>\npeople saw um around 2020 or at least\npeople saw um around 2020 or at least from<00:41:01.880><c> a</c><00:41:02.000><c> long</c><00:41:02.200><c> time</c><00:41:02.359><c> but</c><00:41:02.480><c> they've</c><00:41:02.640><c> been</c><00:41:02.839><c> able</c>\nfrom a long time but they've been able\nfrom a long time but they've been able to<00:41:03.800><c> kind</c><00:41:03.960><c> of</c><00:41:04.480><c> theoretically</c><00:41:05.280><c> show</c><00:41:05.560><c> it</c><00:41:05.920><c> or</c>\nto kind of theoretically show it or\nto kind of theoretically show it or impurely<00:41:06.560><c> show</c><00:41:06.760><c> it</c><00:41:06.920><c> since</c><00:41:07.119><c> 2020</c><00:41:07.880><c> is</c><00:41:08.000><c> that</c><00:41:08.200><c> the</c>\nimpurely show it since 2020 is that the\nimpurely show it since 2020 is that the more<00:41:08.599><c> data</c><00:41:08.880><c> you</c><00:41:08.960><c> train</c><00:41:09.200><c> your</c><00:41:09.319><c> models</c><00:41:09.640><c> on</c><00:41:10.079><c> and</c>\nmore data you train your models on and\nmore data you train your models on and the<00:41:10.280><c> larger</c><00:41:10.599><c> the</c><00:41:10.720><c> models</c><00:41:11.160><c> the</c><00:41:11.280><c> better</c><00:41:11.520><c> the</c>\nthe larger the models the better the\nthe larger the models the better the performance<00:41:12.760><c> this</c><00:41:12.839><c> is</c><00:41:13.000><c> actually</c><00:41:13.200><c> pretty</c>\nperformance this is actually pretty\nperformance this is actually pretty different<00:41:14.079><c> than</c><00:41:14.240><c> what</c><00:41:14.359><c> you've</c><00:41:14.520><c> seen</c><00:41:14.720><c> in</c><00:41:14.880><c> this</c>\ndifferent than what you've seen in this\ndifferent than what you've seen in this class<00:41:15.599><c> in</c><00:41:15.800><c> this</c><00:41:15.960><c> class</c><00:41:16.200><c> we</c><00:41:16.319><c> teach</c><00:41:16.520><c> you</c><00:41:16.680><c> about</c>\nclass in this class we teach you about\nclass in this class we teach you about overfitting<00:41:17.720><c> overfitting</c><00:41:18.400><c> doesn't</c><00:41:18.680><c> happen</c>\noverfitting overfitting doesn't happen\noverfitting overfitting doesn't happen with<00:41:19.119><c> large</c><00:41:19.359><c> language</c><00:41:19.680><c> models</c><00:41:20.599><c> uh</c><00:41:20.839><c> larger</c>\nwith large language models uh larger\nwith large language models uh larger models<00:41:21.880><c> better</c><00:41:22.400><c> performance</c><00:41:23.400><c> um</c><00:41:23.839><c> it's</c>\nmodels better performance um it's\nmodels better performance um it's something<00:41:24.480><c> that</c><00:41:24.680><c> really</c><00:41:24.920><c> took</c><00:41:25.119><c> a</c><00:41:25.280><c> long</c><00:41:25.560><c> time</c>\nsomething that really took a long time\nsomething that really took a long time for<00:41:25.880><c> the</c><00:41:26.040><c> community</c><00:41:26.800><c> who</c><00:41:26.960><c> took</c><00:41:27.440><c> this</c><00:41:27.599><c> type</c><00:41:27.800><c> of</c>\nfor the community who took this type of\nfor the community who took this type of class<00:41:28.440><c> to</c><00:41:28.800><c> realize</c><00:41:29.800><c> um</c><00:41:30.160><c> but</c><00:41:30.319><c> for</c><00:41:30.440><c> the</c><00:41:30.599><c> exam</c>\nclass to realize um but for the exam\nclass to realize um but for the exam overfitting\noverfitting\noverfitting exists<00:41:33.760><c> so</c><00:41:34.680><c> okay</c><00:41:35.040><c> the</c><00:41:35.240><c> idea</c><00:41:35.800><c> of</c><00:41:35.920><c> scaling</c><00:41:36.280><c> laws</c>\nexists so okay the idea of scaling laws\nexists so okay the idea of scaling laws is<00:41:36.880><c> that</c><00:41:37.240><c> if</c><00:41:37.599><c> given</c><00:41:37.839><c> that</c><00:41:37.960><c> you</c><00:41:38.040><c> know</c><00:41:38.240><c> that</c><00:41:38.359><c> more</c>\nis that if given that you know that more\nis that if given that you know that more data<00:41:39.200><c> and</c><00:41:39.359><c> larger</c><00:41:40.000><c> models</c><00:41:40.440><c> will</c><00:41:40.680><c> always</c><00:41:41.040><c> give</c>\ndata and larger models will always give\ndata and larger models will always give you<00:41:41.319><c> better</c><00:41:41.680><c> performance</c><00:41:42.680><c> can</c><00:41:42.839><c> we</c><00:41:43.160><c> predict</c>\nyou better performance can we predict\nyou better performance can we predict how<00:41:44.760><c> much</c><00:41:45.040><c> better</c><00:41:45.319><c> your</c><00:41:45.480><c> performance</c><00:41:46.000><c> will</c><00:41:46.200><c> be</c>\nhow much better your performance will be\nhow much better your performance will be if<00:41:46.800><c> you</c><00:41:47.000><c> increase</c><00:41:47.480><c> the</c><00:41:47.599><c> amount</c><00:41:47.800><c> of</c><00:41:47.960><c> data</c><00:41:48.280><c> and</c>\nif you increase the amount of data and\nif you increase the amount of data and the<00:41:48.560><c> size</c><00:41:48.760><c> of</c><00:41:48.880><c> your</c><00:41:49.440><c> model</c><00:41:50.440><c> and</c><00:41:50.599><c> surprisingly</c>\nthe size of your model and surprisingly\nthe size of your model and surprisingly it<00:41:51.520><c> works</c><00:41:52.520><c> uh</c><00:41:52.640><c> so</c><00:41:52.839><c> here</c><00:41:52.960><c> you</c><00:41:53.079><c> see</c><00:41:53.359><c> three</c><00:41:53.560><c> plots</c>\nit works uh so here you see three plots\nit works uh so here you see three plots from<00:41:54.040><c> a</c><00:41:54.200><c> very</c><00:41:54.400><c> famous</c><00:41:54.720><c> paper</c><00:41:55.079><c> called</c><00:41:55.280><c> scaling</c>\nfrom a very famous paper called scaling\nfrom a very famous paper called scaling loss<00:41:56.000><c> from</c><00:41:56.160><c> openi</c><00:41:57.359><c> um</c><00:41:57.960><c> here</c><00:41:58.079><c> you</c><00:41:58.200><c> see</c><00:41:58.400><c> on</c><00:41:58.520><c> the</c>\nloss from openi um here you see on the\nloss from openi um here you see on the x-axis<00:41:59.319><c> compute</c><00:42:00.000><c> so</c><00:42:00.359><c> how</c><00:42:00.520><c> much</c><00:42:00.680><c> did</c><00:42:00.839><c> you</c><00:42:00.960><c> train</c>\nx-axis compute so how much did you train\nx-axis compute so how much did you train like<00:42:01.760><c> how</c><00:42:01.839><c> much</c><00:42:02.040><c> compute</c><00:42:02.359><c> did</c><00:42:02.520><c> you</c><00:42:02.720><c> did</c><00:42:02.839><c> you</c>\nlike how much compute did you did you\nlike how much compute did you did you spend<00:42:03.160><c> for</c><00:42:03.319><c> training</c><00:42:03.960><c> and</c><00:42:04.160><c> here</c><00:42:04.280><c> you</c><00:42:04.400><c> see</c><00:42:04.599><c> test</c>\nspend for training and here you see test\nspend for training and here you see test loss<00:42:05.319><c> so</c><00:42:05.520><c> this</c><00:42:05.640><c> is</c><00:42:05.920><c> essentially</c><00:42:06.880><c> I</c><00:42:06.920><c> mean</c><00:42:07.040><c> it's</c>\nloss so this is essentially I mean it's\nloss so this is essentially I mean it's not<00:42:07.319><c> perplexity</c><00:42:07.839><c> but</c><00:42:07.920><c> it's</c><00:42:08.000><c> your</c><00:42:08.160><c> validation</c>\nnot perplexity but it's your validation\nnot perplexity but it's your validation loss<00:42:09.319><c> um</c><00:42:09.640><c> so</c><00:42:09.920><c> it's</c><00:42:10.040><c> a</c><00:42:10.160><c> log</c><00:42:10.400><c> of</c><00:42:10.480><c> the</c><00:42:10.640><c> perplexity</c>\nloss um so it's a log of the perplexity\nloss um so it's a log of the perplexity and<00:42:11.839><c> if</c><00:42:11.920><c> you</c><00:42:12.119><c> put</c><00:42:12.400><c> these</c><00:42:12.560><c> two</c><00:42:13.200><c> on</c><00:42:13.720><c> uh</c><00:42:13.839><c> log</c><00:42:14.160><c> scale</c>\nand if you put these two on uh log scale\nand if you put these two on uh log scale uh<00:42:15.200><c> then</c><00:42:15.319><c> you</c><00:42:15.440><c> see</c><00:42:15.760><c> that</c><00:42:16.200><c> uh</c><00:42:16.440><c> the</c><00:42:17.000><c> the</c>\nuh then you see that uh the the\nuh then you see that uh the the performance<00:42:17.920><c> or</c><00:42:18.079><c> like</c><00:42:18.280><c> the</c><00:42:18.480><c> this</c><00:42:18.960><c> the</c><00:42:19.680><c> sorry</c>\nperformance or like the this the sorry\nperformance or like the this the sorry the<00:42:20.640><c> the</c><00:42:20.720><c> scaling</c><00:42:21.160><c> law</c><00:42:21.400><c> is</c><00:42:21.559><c> linear</c><00:42:22.480><c> uh</c><00:42:22.640><c> that</c>\nthe the scaling law is linear uh that\nthe the scaling law is linear uh that means<00:42:23.160><c> that</c><00:42:23.440><c> if</c><00:42:23.559><c> you</c><00:42:23.720><c> increase</c><00:42:24.119><c> your</c><00:42:24.319><c> compute</c>\nmeans that if you increase your compute\nmeans that if you increase your compute by<00:42:25.200><c> a</c><00:42:25.319><c> certain</c><00:42:25.599><c> amount</c><00:42:25.839><c> you</c><00:42:26.000><c> can</c><00:42:26.319><c> you</c><00:42:26.400><c> can</c><00:42:26.559><c> say</c>\nby a certain amount you can you can say\nby a certain amount you can you can say by<00:42:26.880><c> how</c><00:42:27.280><c> much</c><00:42:27.760><c> your</c><00:42:28.040><c> test</c><00:42:28.319><c> loss</c><00:42:28.760><c> will</c><00:42:29.000><c> actually</c>\nby how much your test loss will actually\nby how much your test loss will actually decrease<00:42:30.480><c> same</c><00:42:30.720><c> thing</c><00:42:30.880><c> with</c><00:42:31.119><c> data</c><00:42:31.760><c> and</c><00:42:31.920><c> same</c>\ndecrease same thing with data and same\ndecrease same thing with data and same thing<00:42:32.240><c> for</c><00:42:32.559><c> parameters</c><00:42:33.440><c> if</c><00:42:33.559><c> you</c><00:42:33.760><c> increase</c><00:42:34.160><c> the</c>\nthing for parameters if you increase the\nthing for parameters if you increase the data<00:42:34.559><c> set</c><00:42:34.800><c> size</c><00:42:35.480><c> your</c><00:42:35.680><c> loss</c><00:42:36.040><c> will</c><00:42:36.480><c> will</c>\ndata set size your loss will will\ndata set size your loss will will decrease<00:42:37.480><c> by</c><00:42:37.720><c> an</c><00:42:37.960><c> amount</c><00:42:38.480><c> that</c><00:42:38.720><c> is</c><00:42:38.920><c> somewhat</c>\ndecrease by an amount that is somewhat\ndecrease by an amount that is somewhat predictable<00:42:40.040><c> if</c><00:42:40.160><c> you</c><00:42:40.319><c> increase</c><00:42:40.640><c> the</c><00:42:40.760><c> number</c>\npredictable if you increase the number\npredictable if you increase the number of<00:42:41.040><c> parameters</c><00:42:42.000><c> it</c><00:42:42.119><c> will</c><00:42:42.359><c> decre</c><00:42:42.720><c> the</c><00:42:42.800><c> loss</c>\nof parameters it will decre the loss\nof parameters it will decre the loss will<00:42:43.200><c> decrease</c><00:42:43.599><c> by</c><00:42:43.960><c> amount</c><00:42:44.280><c> which</c><00:42:44.359><c> is</c>\nwill decrease by amount which is\nwill decrease by amount which is somewhat<00:42:44.800><c> predictable</c><00:42:45.760><c> this</c><00:42:45.880><c> is</c><00:42:46.240><c> really</c>\nsomewhat predictable this is really\nsomewhat predictable this is really amazing<00:42:47.599><c> um</c><00:42:48.240><c> very</c><00:42:48.520><c> surprising</c><00:42:49.520><c> I</c><00:42:49.599><c> mean</c><00:42:49.760><c> it</c>\namazing um very surprising I mean it\namazing um very surprising I mean it looks<00:42:50.319><c> in</c><00:42:50.520><c> nocuous</c><00:42:51.040><c> when</c><00:42:51.200><c> you</c><00:42:51.359><c> look</c><00:42:51.480><c> at</c><00:42:51.640><c> these</c>\nlooks in nocuous when you look at these\nlooks in nocuous when you look at these type<00:42:52.000><c> of</c><00:42:52.119><c> plots</c><00:42:52.640><c> but</c><00:42:52.760><c> that's</c><00:42:52.960><c> crazy</c><00:42:53.319><c> because</c>\ntype of plots but that's crazy because\ntype of plots but that's crazy because it<00:42:53.520><c> means</c><00:42:53.720><c> that</c><00:42:53.839><c> you</c><00:42:53.920><c> can</c><00:42:54.119><c> predict</c><00:42:55.119><c> uh</c><00:42:55.359><c> how</c>\nit means that you can predict uh how\nit means that you can predict uh how well<00:42:55.760><c> we're</c><00:42:55.920><c> going</c><00:42:56.000><c> to</c><00:42:56.160><c> perform</c><00:42:56.800><c> in</c><00:42:57.160><c> 2</c><00:42:57.400><c> 3</c><00:42:57.640><c> years</c>\nwell we're going to perform in 2 3 years\nwell we're going to perform in 2 3 years depending<00:42:58.400><c> on</c><00:42:58.559><c> how</c><00:42:58.680><c> much</c><00:42:58.880><c> compute</c><00:42:59.240><c> we</c><00:42:59.359><c> will</c>\ndepending on how much compute we will\ndepending on how much compute we will add<00:43:00.000><c> assuming</c><00:43:00.400><c> that</c><00:43:00.559><c> these</c><00:43:00.720><c> things</c><00:43:01.000><c> will</c><00:43:01.200><c> hold</c>\nadd assuming that these things will hold\nadd assuming that these things will hold there's<00:43:01.680><c> nothing</c><00:43:01.920><c> theoretical</c><00:43:02.440><c> about</c><00:43:02.640><c> it</c><00:43:03.599><c> um</c>\nthere's nothing theoretical about it um\nthere's nothing theoretical about it um yes<00:43:05.800><c> two</c><00:43:06.040><c> things</c><00:43:06.520><c> one</c><00:43:06.839><c> what</c><00:43:06.960><c> is</c><00:43:07.119><c> the</c><00:43:07.240><c> loss</c><00:43:07.480><c> that</c>\nyes two things one what is the loss that\nyes two things one what is the loss that they're<00:43:07.720><c> using</c><00:43:07.960><c> here</c><00:43:08.079><c> is</c><00:43:08.200><c> this</c><00:43:08.400><c> perplexity</c><00:43:09.079><c> or</c>\nthey're using here is this perplexity or\nthey're using here is this perplexity or so<00:43:09.680><c> it's</c><00:43:10.000><c> it's</c><00:43:10.440><c> you</c><00:43:10.559><c> know</c><00:43:10.760><c> I</c><00:43:10.839><c> said</c><00:43:11.040><c> perplexity</c>\nso it's it's you know I said perplexity\nso it's it's you know I said perplexity was<00:43:11.720><c> like</c><00:43:11.880><c> two</c><00:43:12.119><c> to</c><00:43:12.280><c> the</c><00:43:12.400><c> power</c><00:43:12.680><c> of</c><00:43:12.880><c> the</c><00:43:13.000><c> LW</c><00:43:13.319><c> so</c>\nwas like two to the power of the LW so\nwas like two to the power of the LW so this<00:43:13.599><c> is</c><00:43:13.880><c> the</c><00:43:14.559><c> the</c><00:43:14.920><c> the</c><00:43:15.079><c> power</c><00:43:15.880><c> of</c><00:43:16.040><c> the</c>\nthis is the the the power of the\nthis is the the the power of the perplexity<00:43:17.000><c> and</c><00:43:17.119><c> then</c><00:43:17.400><c> the</c><00:43:17.559><c> second</c><00:43:17.880><c> thing</c><00:43:18.280><c> is</c>\nperplexity and then the second thing is\nperplexity and then the second thing is when<00:43:19.040><c> you</c><00:43:19.520><c> like</c><00:43:19.680><c> increase</c><00:43:20.040><c> the</c><00:43:20.119><c> number</c><00:43:20.319><c> of</c>\nwhen you like increase the number of\nwhen you like increase the number of parameters<00:43:20.960><c> or</c><00:43:21.079><c> you</c><00:43:21.240><c> increase</c><00:43:21.520><c> the</c><00:43:21.640><c> total</c>\nparameters or you increase the total\nparameters or you increase the total data<00:43:22.200><c> set</c><00:43:22.400><c> size</c><00:43:22.839><c> going</c><00:43:23.599><c> dat</c><00:43:24.599><c> times</c><00:43:25.000><c> doesn't</c>\ndata set size going dat times doesn't\ndata set size going dat times doesn't that<00:43:25.480><c> just</c><00:43:26.200><c> inherently</c><00:43:26.680><c> increase</c><00:43:26.960><c> your</c>\nthat just inherently increase your\nthat just inherently increase your compute<00:43:27.720><c> like</c><00:43:28.000><c> do</c><00:43:28.160><c> all</c><00:43:28.400><c> this</c><00:43:28.559><c> work</c><00:43:29.040><c> to</c>\ncompute like do all this work to\ncompute like do all this work to just<00:43:32.079><c> specific</c><00:43:32.640><c> no</c><00:43:32.760><c> this</c><00:43:32.839><c> is</c><00:43:32.920><c> a</c><00:43:33.040><c> great</c>\njust specific no this is a great\njust specific no this is a great question<00:43:33.640><c> so</c><00:43:33.800><c> the</c><00:43:33.960><c> compute</c><00:43:34.480><c> here</c><00:43:35.079><c> is</c><00:43:35.240><c> actually</c>\nquestion so the compute here is actually\nquestion so the compute here is actually a<00:43:35.559><c> factor</c><00:43:35.839><c> of</c><00:43:35.960><c> two</c><00:43:36.160><c> things</c><00:43:36.559><c> the</c><00:43:36.760><c> data</c><00:43:37.200><c> and</c><00:43:37.359><c> the</c>\na factor of two things the data and the\na factor of two things the data and the parameter<00:43:38.359><c> what</c><00:43:38.480><c> I'm</c><00:43:38.599><c> showing</c><00:43:38.960><c> here</c><00:43:39.160><c> is</c><00:43:39.280><c> that</c>\nparameter what I'm showing here is that\nparameter what I'm showing here is that you<00:43:39.599><c> can</c><00:43:40.079><c> um</c><00:43:40.319><c> well</c><00:43:40.480><c> actually</c><00:43:40.640><c> we're</c><00:43:40.760><c> going</c><00:43:40.839><c> to</c>\nyou can um well actually we're going to\nyou can um well actually we're going to talk<00:43:41.119><c> about</c><00:43:41.319><c> that</c><00:43:41.440><c> in</c><00:43:41.599><c> details</c><00:43:42.040><c> but</c><00:43:42.200><c> basically</c>\ntalk about that in details but basically\ntalk about that in details but basically if<00:43:43.079><c> you</c><00:43:43.200><c> increase</c><00:43:43.480><c> the</c><00:43:43.559><c> number</c><00:43:43.720><c> of</c><00:43:43.839><c> parameters</c>\nif you increase the number of parameters\nif you increase the number of parameters you<00:43:44.520><c> should</c><00:43:44.720><c> increase</c><00:43:45.040><c> the</c><00:43:45.119><c> number</c><00:43:45.319><c> of</c><00:43:45.520><c> data</c>\nyou should increase the number of data\nyou should increase the number of data that<00:43:46.160><c> you</c><00:43:46.440><c> have</c><00:43:47.480><c> um</c><00:43:48.480><c> so</c><00:43:48.720><c> you</c><00:43:48.880><c> actually</c><00:43:49.040><c> don't</c>\nthat you have um so you actually don't\nthat you have um so you actually don't go<00:43:49.440><c> multiple</c><00:43:49.800><c> times</c><00:43:50.040><c> through</c><00:43:50.200><c> the</c><00:43:50.319><c> same</c><00:43:50.480><c> data</c>\ngo multiple times through the same data\ngo multiple times through the same data set<00:43:51.559><c> no</c><00:43:51.680><c> one</c><00:43:51.960><c> does</c><00:43:52.480><c> EPO</c><00:43:53.480><c> in</c><00:43:54.079><c> a</c><00:43:54.240><c> lar</c><00:43:54.920><c> at</c><00:43:55.000><c> least</c>\nset no one does EPO in a lar at least\nset no one does EPO in a lar at least not<00:43:55.640><c> yet</c><00:43:56.640><c> uh</c><00:43:56.760><c> because</c><00:43:57.119><c> we</c><00:43:57.240><c> have</c><00:43:57.640><c> still</c><00:43:58.359><c> kind</c><00:43:58.520><c> of</c>\nnot yet uh because we have still kind of\nnot yet uh because we have still kind of enough<00:43:59.000><c> data</c><00:43:59.800><c> um</c><00:44:00.079><c> so</c><00:44:00.319><c> yeah</c><00:44:00.480><c> this</c><00:44:00.599><c> is</c><00:44:00.760><c> all</c><00:44:00.960><c> the</c>\nenough data um so yeah this is all the\nenough data um so yeah this is all the same<00:44:01.359><c> Trend</c><00:44:01.720><c> which</c><00:44:01.839><c> is</c><00:44:02.040><c> increase</c><00:44:02.480><c> compute</c>\nsame Trend which is increase compute\nsame Trend which is increase compute decrease\ndecrease\ndecrease loss<00:44:05.200><c> yes</c><00:44:06.040><c> have</c><00:44:06.200><c> we</c><00:44:06.319><c> seen</c><00:44:06.720><c> the</c><00:44:06.920><c> numbers</c><00:44:07.280><c> for</c>\nloss yes have we seen the numbers for\nloss yes have we seen the numbers for the<00:44:07.720><c> last</c><00:44:08.000><c> two</c><00:44:08.240><c> years</c><00:44:09.040><c> or</c><00:44:09.480><c> is</c><00:44:09.640><c> it</c><00:44:09.839><c> still</c>\nthe last two years or is it still\nthe last two years or is it still holding<00:44:11.040><c> it</c><00:44:11.160><c> is</c><00:44:11.359><c> still</c><00:44:11.680><c> holding</c><00:44:12.680><c> I</c><00:44:13.520><c> I</c><00:44:13.640><c> don't</c>\nholding it is still holding I I don't\nholding it is still holding I I don't have<00:44:14.280><c> like</c><00:44:14.520><c> good</c><00:44:14.839><c> numbers</c><00:44:15.240><c> to</c><00:44:15.400><c> show</c><00:44:15.640><c> you</c><00:44:16.480><c> uh</c>\nhave like good numbers to show you uh\nhave like good numbers to show you uh but<00:44:16.760><c> it</c><00:44:16.880><c> is</c><00:44:17.079><c> still</c><00:44:17.319><c> holding</c>\nsurprisingly<00:44:21.280><c> yes</c><00:44:21.800><c> is</c><00:44:21.920><c> there</c><00:44:22.160><c> no</c><00:44:22.359><c> evidence</c>\nsurprisingly yes is there no evidence\nsurprisingly yes is there no evidence like<00:44:22.920><c> empirical</c><00:44:23.359><c> evidence</c><00:44:23.640><c> that</c><00:44:23.720><c> you</c>\nlike empirical evidence that you\nlike empirical evidence that you plateau<00:44:26.280><c> expected</c><00:44:26.640><c> PL</c>\nplateau expected PL\nplateau expected PL no<00:44:29.160><c> empirical</c><00:44:29.680><c> evidence</c><00:44:29.960><c> of</c><00:44:30.119><c> plateauing</c>\nno empirical evidence of plateauing\nno empirical evidence of plateauing anytime<00:44:31.359><c> soon</c><00:44:32.480><c> um</c><00:44:33.480><c> why</c><00:44:34.319><c> we</c><00:44:34.440><c> don't</c><00:44:34.720><c> know</c><00:44:35.720><c> um</c>\nanytime soon um why we don't know um\nanytime soon um why we don't know um will<00:44:36.359><c> it</c><00:44:36.720><c> happen</c><00:44:37.720><c> probably</c><00:44:38.280><c> I</c><00:44:38.319><c> mean</c><00:44:38.480><c> it</c>\nwill it happen probably I mean it\nwill it happen probably I mean it doesn't<00:44:38.760><c> need</c><00:44:38.920><c> to</c><00:44:39.160><c> because</c><00:44:39.319><c> it's</c><00:44:39.440><c> actually</c><00:44:39.599><c> in</c>\ndoesn't need to because it's actually in\ndoesn't need to because it's actually in log<00:44:40.119><c> scale</c><00:44:41.119><c> so</c><00:44:41.440><c> it's</c><00:44:41.680><c> not</c><00:44:42.119><c> like</c><00:44:42.319><c> as</c><00:44:42.440><c> if</c><00:44:42.599><c> it</c><00:44:42.800><c> had</c>\nlog scale so it's not like as if it had\nlog scale so it's not like as if it had to<00:44:43.319><c> go</c><00:44:43.839><c> it</c><00:44:44.040><c> had</c><00:44:44.200><c> to</c><00:44:44.359><c> Plateau</c><00:44:44.839><c> like</c>\nto go it had to Plateau like\nto go it had to Plateau like mathematically<00:44:45.720><c> it</c><00:44:45.839><c> could</c><00:44:46.079><c> continue</c>\nmathematically it could continue\nmathematically it could continue decreasing<00:44:47.000><c> like</c><00:44:47.200><c> this</c><00:44:47.960><c> I</c><00:44:48.000><c> mean</c><00:44:48.200><c> most</c><00:44:48.400><c> people</c>\ndecreasing like this I mean most people\ndecreasing like this I mean most people think<00:44:48.760><c> that</c><00:44:48.880><c> it</c><00:44:48.960><c> will</c><00:44:49.119><c> probably</c><00:44:49.359><c> Plateau</c><00:44:49.720><c> at</c>\nthink that it will probably Plateau at\nthink that it will probably Plateau at some<00:44:50.000><c> point</c><00:44:50.640><c> we</c><00:44:50.720><c> don't</c><00:44:50.880><c> know</c>\nsome point we don't know\nsome point we don't know when<00:44:53.400><c> um</c><00:44:54.400><c> okay</c><00:44:54.720><c> so</c><00:44:54.920><c> that's</c><00:44:55.440><c> I'll</c><00:44:55.680><c> talk</c><00:44:55.839><c> more</c>\nwhen um okay so that's I'll talk more\nwhen um okay so that's I'll talk more about<00:44:56.119><c> scaling</c><00:44:56.400><c> laws</c><00:44:56.720><c> now</c>\nabout scaling laws now\nabout scaling laws now so<00:44:58.079><c> why</c><00:44:58.280><c> are</c><00:44:58.440><c> scaling</c><00:44:58.760><c> laws</c><00:44:59.160><c> really</c><00:44:59.400><c> cool</c>\nso why are scaling laws really cool\nso why are scaling laws really cool imagine<00:45:00.640><c> that</c><00:45:00.839><c> I</c><00:45:01.000><c> give</c><00:45:01.200><c> you</c><00:45:02.040><c> um</c><00:45:02.280><c> you're</c><00:45:02.520><c> very</c>\nimagine that I give you um you're very\nimagine that I give you um you're very fortunate<00:45:03.160><c> I</c><00:45:03.280><c> gave</c><00:45:03.400><c> you</c><00:45:03.520><c> 10,000</c><00:45:04.000><c> gpus</c><00:45:04.480><c> for</c>\nfortunate I gave you 10,000 gpus for\nfortunate I gave you 10,000 gpus for this<00:45:04.920><c> month</c><00:45:05.920><c> what</c><00:45:06.160><c> model</c><00:45:06.480><c> will</c><00:45:06.599><c> you</c><00:45:06.760><c> train</c><00:45:07.400><c> how</c>\nthis month what model will you train how\nthis month what model will you train how do<00:45:07.640><c> you</c><00:45:07.760><c> even</c><00:45:08.000><c> go</c><00:45:08.160><c> about</c><00:45:08.400><c> answering</c><00:45:08.760><c> that</c>\ndo you even go about answering that\ndo you even go about answering that question<00:45:09.800><c> and</c><00:45:10.079><c> I</c><00:45:10.160><c> mean</c><00:45:10.880><c> this</c><00:45:11.000><c> is</c><00:45:11.280><c> a</c><00:45:11.599><c> a</c>\nquestion and I mean this is a a\nquestion and I mean this is a a hypothetical<00:45:12.440><c> but</c><00:45:12.559><c> that's</c><00:45:12.720><c> exactly</c><00:45:13.119><c> what</c>\nhypothetical but that's exactly what\nhypothetical but that's exactly what these<00:45:13.440><c> companies</c><00:45:14.000><c> are</c><00:45:14.240><c> faced</c><00:45:15.079><c> with</c><00:45:16.079><c> uh</c><00:45:16.240><c> the</c>\nthese companies are faced with uh the\nthese companies are faced with uh the old<00:45:16.839><c> pipeline</c><00:45:17.839><c> um</c><00:45:18.319><c> which</c><00:45:18.880><c> was</c><00:45:19.280><c> basically</c><00:45:19.599><c> you</c>\nold pipeline um which was basically you\nold pipeline um which was basically you tune<00:45:19.960><c> High</c><00:45:20.160><c> parameters</c><00:45:20.559><c> on</c><00:45:20.680><c> the</c><00:45:20.760><c> big</c><00:45:21.000><c> models</c>\ntune High parameters on the big models\ntune High parameters on the big models so<00:45:22.000><c> let's</c><00:45:22.160><c> say</c><00:45:22.319><c> I</c><00:45:22.480><c> have</c><00:45:22.880><c> 30</c><00:45:23.240><c> days</c><00:45:23.760><c> I</c><00:45:23.839><c> will</c><00:45:24.040><c> train</c>\nso let's say I have 30 days I will train\nso let's say I have 30 days I will train 30<00:45:24.720><c> models</c><00:45:25.040><c> for</c><00:45:25.240><c> one</c><00:45:25.480><c> day</c><00:45:26.119><c> each</c><00:45:27.000><c> I</c><00:45:27.040><c> will</c><00:45:27.200><c> pick</c>\n30 models for one day each I will pick\n30 models for one day each I will pick the<00:45:27.520><c> best</c><00:45:27.760><c> one</c><00:45:28.640><c> uh</c><00:45:28.760><c> and</c><00:45:28.920><c> that</c><00:45:29.000><c> will</c><00:45:29.160><c> be</c><00:45:29.319><c> the</c>\nthe best one uh and that will be the\nthe best one uh and that will be the final<00:45:29.760><c> model</c><00:45:30.079><c> that</c><00:45:30.200><c> I</c><00:45:30.280><c> will</c><00:45:30.480><c> use</c><00:45:30.680><c> in</c>\nfinal model that I will use in\nfinal model that I will use in production<00:45:32.000><c> um</c><00:45:32.280><c> that</c><00:45:32.400><c> means</c><00:45:32.640><c> that</c><00:45:32.800><c> the</c><00:45:32.920><c> model</c>\nproduction um that means that the model\nproduction um that means that the model that<00:45:33.280><c> I</c><00:45:33.440><c> actually</c><00:45:33.680><c> used</c><00:45:34.160><c> was</c><00:45:34.319><c> only</c><00:45:34.520><c> trained</c>\nthat I actually used was only trained\nthat I actually used was only trained for<00:45:35.119><c> one</c><00:45:35.720><c> day</c><00:45:36.720><c> the</c><00:45:36.880><c> new</c><00:45:37.119><c> pipeline</c><00:45:38.119><c> is</c><00:45:38.240><c> that</c><00:45:38.400><c> you</c>\nfor one day the new pipeline is that you\nfor one day the new pipeline is that you first<00:45:38.880><c> find</c><00:45:39.079><c> a</c><00:45:39.240><c> scaling</c><00:45:39.720><c> recipe</c><00:45:40.319><c> so</c><00:45:40.480><c> you</c><00:45:40.680><c> find</c>\nfirst find a scaling recipe so you find\nfirst find a scaling recipe so you find something<00:45:41.400><c> that</c><00:45:41.559><c> tells</c><00:45:41.839><c> you</c><00:45:42.079><c> for</c><00:45:42.280><c> example</c><00:45:43.160><c> oh</c>\nsomething that tells you for example oh\nsomething that tells you for example oh like<00:45:43.680><c> one</c><00:45:43.880><c> common</c><00:45:44.119><c> thing</c><00:45:44.280><c> is</c><00:45:44.400><c> that</c><00:45:44.559><c> if</c><00:45:44.640><c> you</c>\nlike one common thing is that if you\nlike one common thing is that if you increase<00:45:45.160><c> the</c><00:45:45.280><c> size</c><00:45:45.480><c> of</c><00:45:45.559><c> your</c><00:45:45.680><c> model</c><00:45:45.960><c> you</c>\nincrease the size of your model you\nincrease the size of your model you should<00:45:46.160><c> decrease</c><00:45:46.440><c> your</c><00:45:46.559><c> learning</c><00:45:46.839><c> rate</c><00:45:47.319><c> so</c>\nshould decrease your learning rate so\nshould decrease your learning rate so you<00:45:47.559><c> find</c><00:45:47.720><c> a</c><00:45:47.839><c> scaling</c><00:45:48.240><c> recipe</c><00:45:48.800><c> such</c><00:45:49.040><c> that</c><00:45:49.200><c> you</c>\nyou find a scaling recipe such that you\nyou find a scaling recipe such that you know<00:45:49.720><c> if</c><00:45:49.880><c> I</c><00:45:50.040><c> increase</c><00:45:50.400><c> the</c><00:45:50.839><c> the</c><00:45:51.280><c> the</c><00:45:51.680><c> the</c><00:45:51.839><c> size</c>\nknow if I increase the the the the size\nknow if I increase the the the the size of<00:45:52.160><c> my</c><00:45:52.280><c> model</c><00:45:52.640><c> here's</c><00:45:52.839><c> what</c><00:45:52.960><c> I</c><00:45:53.040><c> should</c><00:45:53.240><c> do</c><00:45:53.440><c> with</c>\nof my model here's what I should do with\nof my model here's what I should do with some<00:45:53.760><c> high</c><00:45:54.319><c> parameters</c><00:45:55.319><c> then</c><00:45:55.440><c> you</c><00:45:55.839><c> tune</c><00:45:56.160><c> your</c>\nsome high parameters then you tune your\nsome high parameters then you tune your high<00:45:56.559><c> parameter</c>\nhigh parameter\nhigh parameter on<00:45:58.280><c> smaller</c><00:45:58.720><c> models</c><00:45:59.400><c> of</c><00:45:59.640><c> different</c><00:45:59.920><c> sizes</c>\non smaller models of different sizes\non smaller models of different sizes let's<00:46:00.880><c> say</c><00:46:01.160><c> I</c><00:46:01.240><c> will</c><00:46:01.440><c> say</c><00:46:01.680><c> for</c><00:46:01.920><c> 3</c><00:46:02.160><c> Days</c><00:46:02.400><c> of</c><00:46:02.559><c> my</c><00:46:02.720><c> 30</c>\nlet's say I will say for 3 Days of my 30\nlet's say I will say for 3 Days of my 30 days<00:46:03.559><c> I</c><00:46:03.640><c> will</c><00:46:03.960><c> train</c><00:46:04.440><c> many</c><00:46:04.680><c> different</c><00:46:05.000><c> models</c>\ndays I will train many different models\ndays I will train many different models and<00:46:05.440><c> I</c><00:46:05.480><c> would</c><00:46:05.640><c> do</c><00:46:05.839><c> highper</c><00:46:06.160><c> parameter</c><00:46:06.520><c> tuning</c>\nand I would do highper parameter tuning\nand I would do highper parameter tuning on<00:46:07.240><c> these</c><00:46:07.400><c> small</c><00:46:07.680><c> models</c><00:46:08.079><c> each</c><00:46:08.200><c> of</c><00:46:08.400><c> different</c>\non these small models each of different\non these small models each of different sizes<00:46:09.520><c> then</c><00:46:09.640><c> I</c><00:46:09.760><c> will</c><00:46:09.960><c> fit</c><00:46:10.240><c> a</c><00:46:10.400><c> scaling</c><00:46:10.760><c> law</c><00:46:11.440><c> and</c>\nsizes then I will fit a scaling law and\nsizes then I will fit a scaling law and try<00:46:11.839><c> to</c><00:46:12.440><c> extrapolate</c><00:46:13.440><c> from</c><00:46:13.720><c> these</c><00:46:13.880><c> smaller</c>\ntry to extrapolate from these smaller\ntry to extrapolate from these smaller models<00:46:15.319><c> which</c><00:46:15.559><c> one</c><00:46:15.800><c> will</c><00:46:16.000><c> be</c><00:46:16.200><c> the</c><00:46:16.400><c> best</c><00:46:17.280><c> if</c><00:46:17.480><c> I</c>\nmodels which one will be the best if I\nmodels which one will be the best if I if<00:46:17.800><c> I</c><00:46:17.920><c> train</c><00:46:18.119><c> it</c><00:46:18.240><c> for</c><00:46:18.440><c> much</c><00:46:18.920><c> longer</c><00:46:19.920><c> or</c><00:46:20.040><c> sorry</c>\nif I train it for much longer or sorry\nif I train it for much longer or sorry if<00:46:20.559><c> I</c><00:46:20.760><c> train</c><00:46:21.040><c> it</c><00:46:21.200><c> for</c><00:46:21.359><c> a</c><00:46:21.520><c> larger</c><00:46:22.079><c> model</c><00:46:23.079><c> and</c>\nif I train it for a larger model and\nif I train it for a larger model and then<00:46:23.359><c> I</c><00:46:23.440><c> will</c><00:46:23.559><c> train</c><00:46:23.800><c> the</c><00:46:23.920><c> final</c><00:46:24.240><c> huge</c><00:46:24.520><c> model</c>\nthen I will train the final huge model\nthen I will train the final huge model for<00:46:25.119><c> 27</c><00:46:25.640><c> days</c><00:46:25.920><c> instead</c><00:46:26.200><c> of</c><00:46:26.359><c> just</c><00:46:26.480><c> one</c><00:46:26.720><c> day</c>\nfor 27 days instead of just one day\nfor 27 days instead of just one day um<00:46:28.319><c> so</c><00:46:28.520><c> the</c><00:46:28.680><c> new</c><00:46:28.920><c> pipeline</c><00:46:29.680><c> is</c><00:46:29.960><c> not</c><00:46:30.599><c> train</c>\num so the new pipeline is not train\num so the new pipeline is not train things<00:46:31.559><c> or</c><00:46:31.760><c> do</c><00:46:31.960><c> high</c><00:46:32.160><c> prity</c><00:46:32.520><c> tuning</c><00:46:32.839><c> on</c><00:46:33.000><c> the</c>\nthings or do high prity tuning on the\nthings or do high prity tuning on the real<00:46:33.520><c> scale</c><00:46:33.800><c> of</c><00:46:33.920><c> the</c><00:46:34.000><c> model</c><00:46:34.240><c> that</c><00:46:34.319><c> you're</c>\nreal scale of the model that you're\nreal scale of the model that you're going<00:46:34.520><c> to</c><00:46:34.599><c> use</c><00:46:34.720><c> in</c><00:46:34.960><c> practice</c><00:46:35.599><c> but</c><00:46:35.760><c> do</c><00:46:35.960><c> things</c>\ngoing to use in practice but do things\ngoing to use in practice but do things on<00:46:36.319><c> smaller</c><00:46:37.160><c> ones</c><00:46:38.160><c> at</c><00:46:38.359><c> different</c><00:46:38.640><c> scales</c><00:46:39.480><c> try</c>\non smaller ones at different scales try\non smaller ones at different scales try to<00:46:40.000><c> predict</c><00:46:40.559><c> how</c><00:46:40.720><c> well</c><00:46:40.920><c> they</c><00:46:41.040><c> will</c><00:46:41.240><c> perform</c>\nto predict how well they will perform\nto predict how well they will perform once<00:46:41.720><c> you</c><00:46:41.880><c> make</c><00:46:42.079><c> them</c><00:46:42.240><c> bigger</c><00:46:43.040><c> I</c><00:46:43.119><c> will</c><00:46:43.359><c> give</c><00:46:43.720><c> I</c>\nonce you make them bigger I will give I\nonce you make them bigger I will give I will<00:46:43.960><c> give</c><00:46:44.079><c> you</c><00:46:44.240><c> a</c><00:46:44.400><c> very</c><00:46:44.599><c> concrete</c><00:46:45.000><c> example</c>\nwill give you a very concrete example\nwill give you a very concrete example right<00:46:45.599><c> now</c><00:46:46.440><c> uh</c><00:46:46.599><c> let's</c><00:46:46.839><c> say</c><00:46:47.319><c> Transformers</c>\nright now uh let's say Transformers\nright now uh let's say Transformers versus<00:46:48.680><c> lstms</c><00:46:49.680><c> let's</c><00:46:49.839><c> say</c><00:46:50.040><c> you</c><00:46:50.480><c> you</c><00:46:50.640><c> have</c>\nversus lstms let's say you you have\nversus lstms let's say you you have these<00:46:50.960><c> 10,000</c><00:46:51.400><c> gpus</c><00:46:51.880><c> you</c><00:46:52.000><c> will</c><00:46:52.200><c> not</c><00:46:52.319><c> sure</c>\nthese 10,000 gpus you will not sure\nthese 10,000 gpus you will not sure which<00:46:52.720><c> one</c><00:46:52.880><c> you</c><00:46:52.960><c> should</c><00:46:53.119><c> be</c><00:46:53.240><c> using</c><00:46:53.559><c> should</c><00:46:53.720><c> I</c>\nwhich one you should be using should I\nwhich one you should be using should I be<00:46:53.960><c> using</c><00:46:54.280><c> Transformer</c><00:46:54.800><c> based</c><00:46:55.000><c> model</c><00:46:55.240><c> or</c><00:46:55.400><c> LCM</c>\nbe using Transformer based model or LCM\nbe using Transformer based model or LCM based<00:46:56.119><c> model</c><00:46:56.680><c> what</c><00:46:56.960><c> I</c><00:46:57.040><c> will</c><00:46:57.160><c> do</c><00:46:57.280><c> is</c><00:46:57.400><c> I</c><00:46:57.480><c> will</c>\nbased model what I will do is I will\nbased model what I will do is I will train<00:46:57.920><c> Transformers</c><00:46:58.920><c> at</c><00:46:59.119><c> different</c><00:46:59.359><c> skills</c>\ntrain Transformers at different skills\ntrain Transformers at different skills so<00:47:00.280><c> here</c><00:47:00.400><c> you</c><00:47:00.520><c> see</c><00:47:00.760><c> different</c><00:47:01.040><c> parameters</c><00:47:01.440><c> on</c>\nso here you see different parameters on\nso here you see different parameters on the<00:47:01.839><c> x-axis</c><00:47:02.760><c> Y</c><00:47:02.920><c> axis</c><00:47:03.200><c> is</c><00:47:03.359><c> my</c><00:47:03.520><c> test</c><00:47:03.760><c> loss</c><00:47:04.400><c> I</c><00:47:04.480><c> will</c>\nthe x-axis Y axis is my test loss I will\nthe x-axis Y axis is my test loss I will then<00:47:04.880><c> train</c><00:47:05.280><c> different</c><00:47:05.920><c> different</c><00:47:06.160><c> lstms</c><00:47:07.160><c> at</c>\nthen train different different lstms at\nthen train different different lstms at different<00:47:07.559><c> scales</c><00:47:08.559><c> once</c><00:47:08.760><c> I</c><00:47:08.920><c> have</c><00:47:09.079><c> these</c>\ndifferent scales once I have these\ndifferent scales once I have these points<00:47:09.920><c> I</c><00:47:10.000><c> will</c><00:47:10.200><c> see</c><00:47:10.559><c> oh</c><00:47:10.720><c> it</c><00:47:10.800><c> kind</c><00:47:10.920><c> of</c><00:47:11.079><c> fits</c><00:47:11.319><c> a</c>\npoints I will see oh it kind of fits a\npoints I will see oh it kind of fits a scaling<00:47:12.119><c> law</c><00:47:12.559><c> I</c><00:47:12.640><c> will</c><00:47:12.880><c> fit</c><00:47:13.040><c> my</c><00:47:13.119><c> scaling</c><00:47:13.440><c> law</c>\nscaling law I will fit my scaling law\nscaling law I will fit my scaling law and<00:47:14.040><c> then</c><00:47:14.160><c> I</c><00:47:14.240><c> will</c><00:47:14.400><c> be</c><00:47:14.520><c> able</c><00:47:14.680><c> to</c><00:47:15.119><c> predict</c><00:47:16.119><c> oh</c><00:47:16.480><c> if</c>\nand then I will be able to predict oh if\nand then I will be able to predict oh if I<00:47:16.839><c> had</c><00:47:17.559><c> 10</c><00:47:17.839><c> times</c><00:47:18.079><c> more</c><00:47:18.280><c> compute</c><00:47:18.800><c> here's</c><00:47:19.040><c> how</c>\nI had 10 times more compute here's how\nI had 10 times more compute here's how well<00:47:19.319><c> I</c><00:47:19.400><c> would</c><00:47:19.640><c> perform</c><00:47:20.119><c> for</c><00:47:20.359><c> the</c><00:47:20.520><c> LM</c><00:47:21.359><c> it's</c>\nwell I would perform for the LM it's\nwell I would perform for the LM it's actually<00:47:21.760><c> slightly</c><00:47:22.079><c> less</c><00:47:22.240><c> linear</c><00:47:22.559><c> for</c><00:47:22.680><c> the</c>\nactually slightly less linear for the\nactually slightly less linear for the lstm<00:47:23.599><c> but</c><00:47:23.800><c> like</c><00:47:23.920><c> you</c><00:47:24.040><c> could</c><00:47:24.319><c> probably</c><00:47:24.599><c> try</c><00:47:24.800><c> to</c>\nlstm but like you could probably try to\nlstm but like you could probably try to predict<00:47:25.520><c> where</c><00:47:25.720><c> you</c><00:47:25.800><c> would</c><00:47:26.000><c> end</c><00:47:26.240><c> up</c><00:47:26.640><c> and</c>\npredict where you would end up and\npredict where you would end up and clearly<00:47:27.200><c> from</c><00:47:27.400><c> this</c><00:47:27.559><c> plot</c><00:47:28.200><c> you</c><00:47:28.319><c> would</c><00:47:28.440><c> see</c>\nclearly from this plot you would see\nclearly from this plot you would see that<00:47:28.720><c> Transformers</c><00:47:29.200><c> are</c><00:47:29.599><c> better</c><00:47:30.599><c> um</c><00:47:30.920><c> one</c>\nthat Transformers are better um one\nthat Transformers are better um one thing<00:47:31.240><c> to</c><00:47:31.400><c> notice</c><00:47:31.720><c> when</c><00:47:31.839><c> you</c><00:47:31.960><c> read</c><00:47:32.240><c> these</c><00:47:32.440><c> type</c>\nthing to notice when you read these type\nthing to notice when you read these type of<00:47:32.760><c> scaling</c><00:47:33.079><c> laws</c><00:47:33.359><c> is</c><00:47:33.480><c> that</c><00:47:33.640><c> are</c><00:47:33.800><c> two</c><00:47:33.960><c> things</c>\nof scaling laws is that are two things\nof scaling laws is that are two things that<00:47:34.240><c> are</c><00:47:34.599><c> important</c><00:47:35.599><c> uh</c><00:47:36.000><c> one</c><00:47:36.880><c> is</c><00:47:37.680><c> really</c><00:47:38.000><c> your</c>\nthat are important uh one is really your\nthat are important uh one is really your scaling<00:47:38.960><c> rate</c><00:47:39.960><c> uh</c><00:47:40.119><c> which</c><00:47:40.280><c> is</c><00:47:40.480><c> kind</c><00:47:40.640><c> of</c><00:47:41.119><c> the</c><00:47:42.119><c> uh</c>\nscaling rate uh which is kind of the uh\nscaling rate uh which is kind of the uh the<00:47:42.480><c> slope</c><00:47:43.480><c> of</c><00:47:43.800><c> the</c><00:47:44.319><c> the</c><00:47:44.440><c> slope</c><00:47:44.800><c> of</c><00:47:44.920><c> the</c>\nthe slope of the the slope of the\nthe slope of the the slope of the scaling<00:47:45.359><c> law</c><00:47:45.800><c> the</c><00:47:45.960><c> other</c><00:47:46.160><c> thing</c><00:47:46.400><c> is</c><00:47:46.800><c> your</c><00:47:47.800><c> um</c>\nscaling law the other thing is your um\nscaling law the other thing is your um your<00:47:49.040><c> intercept</c><00:47:50.040><c> like</c><00:47:50.160><c> you</c><00:47:50.280><c> could</c><00:47:50.480><c> start</c>\nyour intercept like you could start\nyour intercept like you could start worse<00:47:51.640><c> but</c><00:47:51.880><c> actually</c><00:47:52.119><c> become</c><00:47:52.480><c> better</c><00:47:52.839><c> over</c>\nworse but actually become better over\nworse but actually become better over time<00:47:53.640><c> it</c><00:47:53.760><c> just</c><00:47:53.920><c> happens</c><00:47:54.160><c> that</c><00:47:54.359><c> lstms</c><00:47:54.839><c> are</c>\ntime it just happens that lstms are\ntime it just happens that lstms are worse<00:47:55.200><c> for</c><00:47:55.400><c> both</c><00:47:56.079><c> uh</c><00:47:56.160><c> but</c><00:47:56.280><c> I</c><00:47:56.359><c> could</c><00:47:56.520><c> show</c><00:47:56.680><c> you</c>\nworse for both uh but I could show you\nworse for both uh but I could show you another<00:47:57.280><c> one</c><00:47:57.839><c> where</c><00:47:58.119><c> things</c><00:47:58.720><c> you</c><00:47:58.839><c> can</c><00:47:59.079><c> predict</c>\nanother one where things you can predict\nanother one where things you can predict that<00:47:59.720><c> actually</c><00:48:00.440><c> after</c><00:48:00.640><c> a</c><00:48:00.760><c> certain</c><00:48:01.079><c> scale</c>\nthat actually after a certain scale\nthat actually after a certain scale you're<00:48:01.559><c> better</c><00:48:01.880><c> off</c><00:48:02.280><c> using</c><00:48:02.640><c> that</c><00:48:02.800><c> type</c><00:48:02.960><c> of</c>\nyou're better off using that type of\nyou're better off using that type of model<00:48:03.400><c> than</c><00:48:03.599><c> others</c><00:48:04.319><c> uh</c><00:48:04.400><c> so</c><00:48:04.559><c> that's</c><00:48:04.760><c> why</c>\nmodel than others uh so that's why\nmodel than others uh so that's why scaling<00:48:05.440><c> laws</c><00:48:06.040><c> are</c><00:48:06.240><c> actually</c><00:48:06.520><c> really</c>\nscaling laws are actually really\nscaling laws are actually really useful<00:48:08.680><c> any</c><00:48:08.880><c> questions</c><00:48:09.160><c> on</c>\nthat<00:48:12.440><c> yeah</c><00:48:13.040><c> so</c><00:48:13.640><c> these</c><00:48:13.800><c> are</c><00:48:13.960><c> all</c><00:48:14.200><c> kind</c><00:48:14.319><c> of</c><00:48:14.800><c> very</c>\nthat yeah so these are all kind of very\nthat yeah so these are all kind of very how<00:48:15.920><c> how</c><00:48:16.160><c> sensitive</c><00:48:16.520><c> are</c><00:48:16.720><c> these</c><00:48:16.880><c> to</c><00:48:17.079><c> like</c>\nhow how sensitive are these to like\nhow how sensitive are these to like small<00:48:17.559><c> differences</c><00:48:17.960><c> in</c><00:48:18.079><c> the</c><00:48:18.240><c> architecture</c>\nsmall differences in the architecture\nsmall differences in the architecture like<00:48:19.960><c> one</c><00:48:20.520><c> one</c><00:48:20.839><c> like</c><00:48:21.000><c> Transformer</c>\nlike one one like Transformer\nlike one one like Transformer architecture<00:48:22.000><c> versus</c><00:48:22.319><c> another</c><00:48:22.599><c> Transformer</c>\narchitecture versus another Transformer\narchitecture versus another Transformer architecture<00:48:23.720><c> you</c><00:48:23.920><c> basically</c><00:48:24.280><c> have</c><00:48:24.400><c> to</c><00:48:24.599><c> like</c>\narchitecture you basically have to like\narchitecture you basically have to like fit<00:48:25.359><c> your</c><00:48:25.559><c> own</c><00:48:25.920><c> curve</c><00:48:26.480><c> and</c><00:48:26.599><c> make</c><00:48:26.760><c> basically</c>\nfit your own curve and make basically\nfit your own curve and make basically say<00:48:27.240><c> like</c><00:48:27.359><c> oh</c><00:48:27.520><c> scaling</c><00:48:27.880><c> law</c><00:48:28.000><c> has</c><00:48:28.079><c> tell</c><00:48:28.240><c> me</c>\nsay like oh scaling law has tell me\nsay like oh scaling law has tell me there<00:48:28.559><c> should</c><00:48:28.800><c> be</c><00:48:29.440><c> some</c><00:48:29.760><c> like</c><00:48:29.920><c> logarithmic</c>\nthere should be some like logarithmic\nthere should be some like logarithmic function<00:48:31.480><c> let</c><00:48:31.640><c> me</c><00:48:32.640><c> extrapolate</c><00:48:33.240><c> that</c><00:48:33.400><c> for</c><00:48:33.599><c> my</c>\nfunction let me extrapolate that for my\nfunction let me extrapolate that for my own<00:48:35.480><c> yeah</c><00:48:35.760><c> so</c><00:48:36.599><c> uh</c><00:48:36.720><c> usually</c><00:48:37.040><c> for</c><00:48:37.200><c> example</c><00:48:37.480><c> if</c>\nown yeah so uh usually for example if\nown yeah so uh usually for example if you're<00:48:37.640><c> an</c><00:48:37.760><c> academic</c><00:48:38.200><c> and</c><00:48:38.280><c> you</c><00:48:38.400><c> want</c><00:48:38.520><c> to</c><00:48:38.800><c> now</c>\nyou're an academic and you want to now\nyou're an academic and you want to now at<00:48:39.119><c> least</c><00:48:39.319><c> that's</c><00:48:39.559><c> like</c><00:48:39.800><c> pretty</c><00:48:40.520><c> recent</c><00:48:41.040><c> and</c>\nat least that's like pretty recent and\nat least that's like pretty recent and you<00:48:41.240><c> want</c><00:48:41.359><c> to</c><00:48:41.559><c> propose</c><00:48:41.839><c> a</c><00:48:41.960><c> new</c><00:48:42.240><c> like</c>\nyou want to propose a new like\nyou want to propose a new like activation<00:48:43.839><c> uh</c><00:48:44.000><c> that's</c><00:48:44.160><c> exactly</c><00:48:44.480><c> what</c><00:48:44.559><c> you</c>\nactivation uh that's exactly what you\nactivation uh that's exactly what you will<00:48:44.800><c> do</c><00:48:45.000><c> you</c><00:48:45.119><c> will</c><00:48:45.359><c> fit</c><00:48:45.520><c> a</c><00:48:45.599><c> scaling</c><00:48:45.920><c> law</c><00:48:46.359><c> show</c>\nwill do you will fit a scaling law show\nwill do you will fit a scaling law show another<00:48:46.920><c> scaling</c><00:48:47.280><c> law</c><00:48:47.520><c> with</c><00:48:47.680><c> the</c><00:48:47.839><c> standard</c>\nanother scaling law with the standard\nanother scaling law with the standard like<00:48:48.480><c> I</c><00:48:48.520><c> don't</c><00:48:48.640><c> know</c><00:48:48.839><c> G</c><00:48:49.559><c> and</c><00:48:49.680><c> you</c><00:48:49.760><c> will</c><00:48:49.880><c> say</c>\nlike I don't know G and you will say\nlike I don't know G and you will say that<00:48:50.200><c> it's</c><00:48:50.400><c> better</c><00:48:51.040><c> in</c><00:48:51.200><c> reality</c><00:48:51.559><c> once</c><00:48:51.720><c> you</c>\nthat it's better in reality once you\nthat it's better in reality once you start<00:48:52.079><c> thinking</c><00:48:52.319><c> about</c><00:48:52.480><c> it</c><00:48:52.599><c> in</c><00:48:52.720><c> scaling</c><00:48:53.079><c> loss</c>\nstart thinking about it in scaling loss\nstart thinking about it in scaling loss terms<00:48:53.960><c> you</c><00:48:54.160><c> really</c><00:48:54.440><c> realize</c><00:48:54.880><c> that</c><00:48:55.240><c> actually</c>\nterms you really realize that actually\nterms you really realize that actually all<00:48:56.079><c> the</c><00:48:56.200><c> architecture</c><00:48:56.760><c> differences</c><00:48:57.079><c> that</c><00:48:57.200><c> we</c>\nall the architecture differences that we\nall the architecture differences that we can<00:48:57.440><c> make</c><00:48:57.640><c> like</c><00:48:57.760><c> the</c><00:48:57.880><c> small</c><00:48:58.160><c> minor</c><00:48:58.520><c> ones</c><00:48:59.119><c> all</c>\ncan make like the small minor ones all\ncan make like the small minor ones all they<00:48:59.480><c> do</c><00:48:59.680><c> is</c><00:48:59.799><c> maybe</c><00:49:00.079><c> change</c><00:49:00.359><c> a</c><00:49:00.520><c> little</c><00:49:00.799><c> bit</c><00:49:01.119><c> the</c>\nthey do is maybe change a little bit the\nthey do is maybe change a little bit the The\nThe\nThe Intercept<00:49:03.440><c> but</c><00:49:03.640><c> really</c><00:49:03.839><c> that</c><00:49:04.000><c> doesn't</c><00:49:04.280><c> matter</c>\nIntercept but really that doesn't matter\nIntercept but really that doesn't matter uh<00:49:05.280><c> cuz</c><00:49:05.440><c> just</c><00:49:05.599><c> train</c><00:49:05.839><c> it</c><00:49:05.960><c> for</c><00:49:06.119><c> 10</c><00:49:06.319><c> hours</c><00:49:06.559><c> longer</c>\nuh cuz just train it for 10 hours longer\nuh cuz just train it for 10 hours longer or<00:49:07.400><c> like</c><00:49:07.640><c> wait</c><00:49:07.880><c> for</c><00:49:08.079><c> the</c><00:49:08.240><c> next</c><00:49:08.680><c> uh</c><00:49:08.839><c> for</c><00:49:09.000><c> the</c>\nor like wait for the next uh for the\nor like wait for the next uh for the next<00:49:09.359><c> Compu</c><00:49:09.760><c> gpus</c><00:49:10.359><c> and</c><00:49:10.520><c> these</c><00:49:10.680><c> things</c><00:49:10.920><c> are</c>\nnext Compu gpus and these things are\nnext Compu gpus and these things are really<00:49:11.480><c> secondary</c><00:49:12.079><c> which</c><00:49:12.200><c> is</c><00:49:12.319><c> exactly</c><00:49:12.640><c> why</c><00:49:12.760><c> I</c>\nreally secondary which is exactly why I\nreally secondary which is exactly why I was<00:49:12.960><c> telling</c><00:49:13.200><c> you</c><00:49:13.359><c> originally</c><00:49:14.000><c> people</c><00:49:14.280><c> spend</c>\nwas telling you originally people spend\nwas telling you originally people spend too<00:49:14.680><c> much</c><00:49:14.839><c> time</c><00:49:14.960><c> on</c><00:49:15.119><c> the</c><00:49:15.280><c> architecture</c><00:49:15.720><c> and</c>\ntoo much time on the architecture and\ntoo much time on the architecture and losses<00:49:16.799><c> um</c><00:49:17.240><c> in</c><00:49:17.400><c> reality</c><00:49:17.799><c> these</c><00:49:17.920><c> things</c><00:49:18.160><c> don't</c>\nlosses um in reality these things don't\nlosses um in reality these things don't matter<00:49:18.559><c> as</c><00:49:18.720><c> much</c><00:49:19.079><c> data</c><00:49:19.520><c> though</c><00:49:19.880><c> if</c><00:49:19.960><c> you</c><00:49:20.079><c> use</c>\nmatter as much data though if you use\nmatter as much data though if you use good<00:49:20.520><c> data</c><00:49:21.079><c> you</c><00:49:21.200><c> will</c><00:49:21.440><c> have</c><00:49:21.799><c> much</c><00:49:22.079><c> better</c>\ngood data you will have much better\ngood data you will have much better scaling<00:49:22.680><c> loss</c><00:49:23.119><c> than</c><00:49:23.240><c> if</c><00:49:23.440><c> use</c><00:49:23.760><c> bad</c><00:49:23.960><c> data</c><00:49:24.359><c> so</c>\nscaling loss than if use bad data so\nscaling loss than if use bad data so that<00:49:24.799><c> really</c><00:49:25.079><c> matters</c>\nthat really matters\nthat really matters uh<00:49:27.520><c> another</c><00:49:27.880><c> really</c><00:49:28.079><c> cool</c><00:49:28.280><c> thing</c><00:49:28.440><c> you</c><00:49:28.520><c> can</c><00:49:28.640><c> do</c>\nuh another really cool thing you can do\nuh another really cool thing you can do with<00:49:28.880><c> scaling</c><00:49:29.200><c> laws</c><00:49:29.720><c> is</c><00:49:29.880><c> that</c><00:49:30.000><c> you</c><00:49:30.079><c> can</c><00:49:30.280><c> ask</c>\nwith scaling laws is that you can ask\nwith scaling laws is that you can ask yourself<00:49:31.559><c> uh</c><00:49:32.160><c> how</c><00:49:32.359><c> to</c><00:49:32.760><c> optimally</c><00:49:33.400><c> allocate</c>\nyourself uh how to optimally allocate\nyourself uh how to optimally allocate training<00:49:34.240><c> resources</c><00:49:35.079><c> should</c><00:49:35.319><c> I</c><00:49:35.559><c> train</c><00:49:35.960><c> larger</c>\ntraining resources should I train larger\ntraining resources should I train larger models<00:49:37.000><c> because</c><00:49:37.119><c> we</c><00:49:37.240><c> saw</c><00:49:37.520><c> that</c><00:49:37.960><c> it's</c><00:49:38.160><c> better</c>\nmodels because we saw that it's better\nmodels because we saw that it's better when<00:49:38.520><c> you</c><00:49:38.599><c> train</c><00:49:38.880><c> larger</c><00:49:39.119><c> models</c><00:49:39.640><c> but</c><00:49:39.799><c> we</c><00:49:39.920><c> saw</c>\nwhen you train larger models but we saw\nwhen you train larger models but we saw that<00:49:40.240><c> it's</c><00:49:40.359><c> also</c><00:49:40.559><c> better</c><00:49:40.799><c> when</c><00:49:40.920><c> you</c><00:49:41.040><c> use</c><00:49:41.480><c> more</c>\nthat it's also better when you use more\nthat it's also better when you use more data<00:49:42.319><c> so</c><00:49:42.680><c> which</c><00:49:42.920><c> one</c><00:49:43.079><c> should</c><00:49:43.240><c> I</c><00:49:43.400><c> do</c><00:49:43.720><c> should</c><00:49:43.880><c> I</c>\ndata so which one should I do should I\ndata so which one should I do should I just<00:49:44.160><c> train</c><00:49:44.400><c> on</c><00:49:44.520><c> more</c><00:49:44.720><c> data</c><00:49:45.040><c> a</c><00:49:45.160><c> smaller</c><00:49:45.480><c> model</c>\njust train on more data a smaller model\njust train on more data a smaller model or<00:49:46.000><c> should</c><00:49:46.160><c> I</c><00:49:46.319><c> train</c><00:49:46.559><c> a</c><00:49:46.680><c> larger</c><00:49:46.960><c> model</c><00:49:47.240><c> on</c><00:49:47.440><c> less</c>\nor should I train a larger model on less\nor should I train a larger model on less data<00:49:48.880><c> um</c><00:49:49.760><c> so</c><00:49:50.760><c> chinchilla</c><00:49:51.319><c> is</c><00:49:51.400><c> a</c><00:49:51.520><c> very</c><00:49:51.720><c> famous</c>\ndata um so chinchilla is a very famous\ndata um so chinchilla is a very famous paper<00:49:52.319><c> that</c><00:49:52.520><c> first</c><00:49:52.720><c> showed</c><00:49:53.119><c> this</c><00:49:53.799><c> uh</c><00:49:53.960><c> the</c><00:49:54.079><c> way</c>\npaper that first showed this uh the way\npaper that first showed this uh the way they<00:49:54.440><c> did</c><00:49:54.599><c> it</c><00:49:55.079><c> I</c><00:49:55.160><c> want</c><00:49:55.280><c> to</c><00:49:55.440><c> give</c><00:49:55.520><c> you</c><00:49:55.640><c> a</c><00:49:55.760><c> little</c>\nthey did it I want to give you a little\nthey did it I want to give you a little bit<00:49:56.280><c> of</c><00:49:56.440><c> a</c><00:49:56.720><c> sense</c><00:49:56.880><c> of</c><00:49:57.000><c> what</c><00:49:57.079><c> these</c><00:49:57.240><c> plots</c><00:49:57.559><c> are</c>\nbit of a sense of what these plots are\nbit of a sense of what these plots are uh<00:49:58.480><c> here</c><00:49:58.599><c> you</c><00:49:58.720><c> see</c><00:49:58.839><c> training</c><00:49:59.119><c> loss</c><00:49:59.520><c> again</c><00:49:59.880><c> on</c>\nuh here you see training loss again on\nuh here you see training loss again on the<00:50:00.160><c> x-axis</c><00:50:00.720><c> you</c><00:50:00.799><c> see</c><00:50:01.079><c> parameter</c><00:50:01.799><c> parameter</c>\nthe x-axis you see parameter parameter\nthe x-axis you see parameter parameter differences<00:50:02.799><c> uh</c><00:50:02.920><c> sorry</c><00:50:03.160><c> parameter</c><00:50:03.520><c> size</c><00:50:03.960><c> uh</c>\ndifferences uh sorry parameter size uh\ndifferences uh sorry parameter size uh number<00:50:04.280><c> of</c><00:50:04.400><c> parameters</c><00:50:04.799><c> so</c><00:50:04.960><c> the</c><00:50:05.079><c> size</c><00:50:05.240><c> of</c><00:50:05.359><c> the</c>\nnumber of parameters so the size of the\nnumber of parameters so the size of the model<00:50:06.280><c> and</c><00:50:06.520><c> here</c><00:50:06.799><c> all</c><00:50:07.000><c> these</c><00:50:07.240><c> curves</c><00:50:07.559><c> are</c><00:50:07.720><c> what</c>\nmodel and here all these curves are what\nmodel and here all these curves are what we<00:50:08.000><c> call</c><00:50:08.240><c> isof</c><00:50:08.680><c> flops</c><00:50:09.319><c> which</c><00:50:09.480><c> is</c><00:50:10.200><c> that</c><00:50:10.760><c> all</c><00:50:11.319><c> the</c>\nwe call isof flops which is that all the\nwe call isof flops which is that all the models<00:50:12.079><c> on</c><00:50:12.400><c> this</c><00:50:12.720><c> curve</c><00:50:13.720><c> H</c><00:50:14.000><c> have</c><00:50:14.160><c> been</c><00:50:14.319><c> trained</c>\nmodels on this curve H have been trained\nmodels on this curve H have been trained with<00:50:14.839><c> the</c><00:50:14.920><c> same</c><00:50:15.119><c> amount</c><00:50:15.359><c> of</c>\nwith the same amount of\nwith the same amount of compute<00:50:17.000><c> um</c><00:50:17.240><c> the</c><00:50:17.359><c> way</c><00:50:17.520><c> that</c><00:50:17.640><c> you</c><00:50:17.799><c> do</c><00:50:18.000><c> that</c><00:50:18.280><c> is</c>\ncompute um the way that you do that is\ncompute um the way that you do that is that<00:50:18.520><c> you</c><00:50:18.640><c> train</c><00:50:19.200><c> you</c><00:50:19.559><c> change</c><00:50:20.119><c> sorry</c><00:50:20.400><c> you</c><00:50:20.520><c> vary</c>\nthat you train you change sorry you vary\nthat you train you change sorry you vary the<00:50:20.880><c> number</c><00:50:21.079><c> of</c><00:50:21.200><c> tokens</c><00:50:21.520><c> that</c><00:50:21.640><c> we</c><00:50:21.839><c> trained</c><00:50:22.119><c> on</c>\nthe number of tokens that we trained on\nthe number of tokens that we trained on and<00:50:22.720><c> the</c><00:50:22.880><c> size</c><00:50:23.079><c> of</c><00:50:23.200><c> the</c><00:50:23.319><c> models</c><00:50:23.920><c> but</c><00:50:24.040><c> you</c><00:50:24.200><c> vary</c>\nand the size of the models but you vary\nand the size of the models but you vary in<00:50:24.559><c> such</c><00:50:24.720><c> a</c><00:50:24.799><c> way</c><00:50:24.960><c> that</c><00:50:25.079><c> the</c><00:50:25.240><c> total</c><00:50:25.559><c> compute</c><00:50:26.119><c> is</c>\nin such a way that the total compute is\nin such a way that the total compute is constant\nconstant\nconstant okay<00:50:27.640><c> so</c><00:50:27.920><c> all</c><00:50:28.079><c> these</c><00:50:28.319><c> curves</c><00:50:28.599><c> that</c><00:50:28.680><c> you</c><00:50:28.760><c> see</c>\nokay so all these curves that you see\nokay so all these curves that you see with<00:50:29.079><c> different</c><00:50:29.280><c> colors</c><00:50:30.040><c> have</c><00:50:30.319><c> different</c>\nwith different colors have different\nwith different colors have different amount<00:50:30.880><c> of</c><00:50:31.000><c> computers</c><00:50:31.400><c> that</c><00:50:31.520><c> were</c><00:50:31.680><c> trained</c><00:50:31.960><c> on</c>\namount of computers that were trained on\namount of computers that were trained on then<00:50:32.880><c> you</c><00:50:33.040><c> take</c><00:50:33.200><c> the</c><00:50:33.359><c> best</c><00:50:33.599><c> one</c><00:50:33.839><c> for</c><00:50:34.079><c> each</c><00:50:34.240><c> of</c>\nthen you take the best one for each of\nthen you take the best one for each of those<00:50:34.599><c> curves</c><00:50:35.559><c> once</c><00:50:35.720><c> you</c><00:50:35.880><c> have</c><00:50:36.000><c> the</c><00:50:36.200><c> best</c><00:50:36.359><c> one</c>\nthose curves once you have the best one\nthose curves once you have the best one for<00:50:36.720><c> each</c><00:50:36.880><c> of</c><00:50:37.040><c> those</c><00:50:37.400><c> curves</c><00:50:38.400><c> um</c><00:50:38.880><c> you</c><00:50:39.319><c> can</c><00:50:40.319><c> ask</c>\nfor each of those curves um you can ask\nfor each of those curves um you can ask you<00:50:40.839><c> can</c><00:50:41.400><c> plot</c><00:50:42.400><c> um</c><00:50:42.880><c> how</c><00:50:43.000><c> much</c><00:50:43.200><c> flops</c><00:50:43.640><c> it</c><00:50:43.799><c> was</c>\nyou can plot um how much flops it was\nyou can plot um how much flops it was and<00:50:44.400><c> which</c><00:50:44.599><c> curve</c><00:50:44.880><c> were</c><00:50:45.040><c> you</c><00:50:45.200><c> on</c><00:50:45.799><c> and</c><00:50:46.000><c> how</c><00:50:46.119><c> much</c>\nand which curve were you on and how much\nand which curve were you on and how much parameters<00:50:47.480><c> did</c><00:50:47.640><c> you</c><00:50:47.880><c> actually</c><00:50:48.200><c> use</c><00:50:48.920><c> for</c>\nparameters did you actually use for\nparameters did you actually use for training<00:50:49.640><c> that</c><00:50:49.880><c> specific</c><00:50:50.280><c> point</c><00:50:50.839><c> you</c><00:50:51.040><c> put</c>\ntraining that specific point you put\ntraining that specific point you put that<00:50:51.839><c> on</c><00:50:52.040><c> the</c><00:50:52.440><c> on</c><00:50:52.559><c> the</c><00:50:52.720><c> log</c><00:50:53.040><c> log</c><00:50:53.760><c> uh</c><00:50:53.920><c> scale</c>\nthat on the on the log log uh scale\nthat on the on the log log uh scale again<00:50:54.760><c> and</c><00:50:54.880><c> now</c><00:50:55.000><c> you</c><00:50:55.119><c> fit</c><00:50:55.319><c> a</c><00:50:55.480><c> scaling</c><00:50:55.839><c> law</c>\nagain and now you fit a scaling law\nagain and now you fit a scaling law again<00:50:56.960><c> so</c><00:50:57.200><c> now</c><00:50:57.400><c> I</c><00:50:57.599><c> have</c><00:50:58.319><c> something</c><00:50:58.760><c> which</c>\nagain so now I have something which\nagain so now I have something which tells<00:50:59.240><c> me</c><00:50:59.920><c> if</c><00:51:00.119><c> I</c><00:51:00.200><c> want</c><00:51:00.359><c> to</c><00:51:00.520><c> train</c><00:51:00.839><c> a</c><00:51:00.960><c> model</c><00:51:01.240><c> of</c>\ntells me if I want to train a model of\ntells me if I want to train a model of 10^<00:51:02.040><c> 23</c><00:51:02.480><c> flops</c><00:51:03.440><c> here's</c><00:51:03.760><c> exactly</c><00:51:04.119><c> the</c><00:51:04.240><c> number</c>\n10^ 23 flops here's exactly the number\n10^ 23 flops here's exactly the number of<00:51:04.599><c> parameters</c><00:51:04.960><c> that</c><00:51:05.079><c> I</c><00:51:05.160><c> should</c><00:51:05.319><c> be</c><00:51:05.440><c> using</c><00:51:06.240><c> 100</c>\nof parameters that I should be using 100\nof parameters that I should be using 100 100b<00:51:07.960><c> and</c><00:51:08.079><c> you</c><00:51:08.160><c> can</c><00:51:08.280><c> do</c><00:51:08.440><c> the</c><00:51:08.559><c> same</c><00:51:08.760><c> thing</c><00:51:08.920><c> with</c>\n100b and you can do the same thing with\n100b and you can do the same thing with flops<00:51:09.599><c> and</c>\nflops and\nflops and tokens<00:51:11.520><c> so</c><00:51:11.680><c> now</c><00:51:11.799><c> you</c><00:51:11.920><c> can</c><00:51:12.559><c> predict</c><00:51:13.559><c> if</c><00:51:13.799><c> if</c><00:51:13.960><c> I</c>\ntokens so now you can predict if if I\ntokens so now you can predict if if I tell<00:51:14.280><c> you</c><00:51:14.480><c> exactly</c><00:51:14.799><c> I</c><00:51:14.920><c> have</c><00:51:15.040><c> one</c><00:51:15.200><c> month</c><00:51:15.440><c> of</c>\ntell you exactly I have one month of\ntell you exactly I have one month of compute<00:51:17.040><c> what</c><00:51:17.240><c> size</c><00:51:17.440><c> of</c><00:51:17.599><c> model</c><00:51:17.839><c> should</c><00:51:18.000><c> I</c><00:51:18.079><c> be</c>\ncompute what size of model should I be\ncompute what size of model should I be training<00:51:18.839><c> F</c><00:51:19.119><c> your</c><00:51:19.240><c> scaling</c><00:51:19.599><c> law</c><00:51:19.880><c> and</c><00:51:20.000><c> I</c><00:51:20.119><c> tell</c>\ntraining F your scaling law and I tell\ntraining F your scaling law and I tell you<00:51:21.599><c> um</c><00:51:22.119><c> of</c><00:51:22.240><c> course</c><00:51:22.440><c> that</c><00:51:22.599><c> all</c><00:51:22.760><c> looks</c>\nyou um of course that all looks\nyou um of course that all looks beautiful<00:51:23.760><c> in</c><00:51:23.960><c> reality</c><00:51:24.520><c> like</c><00:51:24.760><c> there's</c><00:51:25.119><c> like</c>\nbeautiful in reality like there's like\nbeautiful in reality like there's like there's<00:51:25.400><c> a</c><00:51:25.559><c> lot</c><00:51:25.680><c> of</c><00:51:25.880><c> like</c><00:51:26.000><c> small</c><00:51:26.280><c> things</c><00:51:26.680><c> of</c>\nthere's a lot of like small things of\nthere's a lot of like small things of like<00:51:26.920><c> should</c><00:51:27.040><c> you</c><00:51:27.160><c> be</c><00:51:27.319><c> counting</c><00:51:27.720><c> like</c>\nlike should you be counting like\nlike should you be counting like embedding<00:51:28.319><c> parameters</c><00:51:29.160><c> like</c><00:51:29.359><c> there's</c>\nembedding parameters like there's\nembedding parameters like there's there's<00:51:29.680><c> a</c><00:51:29.799><c> lot</c><00:51:29.880><c> of</c><00:51:30.040><c> complexities</c><00:51:31.040><c> but</c><00:51:31.200><c> if</c><00:51:31.280><c> you</c>\nthere's a lot of complexities but if you\nthere's a lot of complexities but if you do<00:51:31.640><c> things</c><00:51:31.920><c> well</c><00:51:32.319><c> these</c><00:51:32.480><c> things</c><00:51:32.720><c> actually</c><00:51:33.000><c> do</c>\ndo things well these things actually do\ndo things well these things actually do hold<00:51:35.000><c> um</c><00:51:35.640><c> so</c><00:51:35.960><c> the</c><00:51:36.240><c> optimal</c><00:51:36.760><c> number</c><00:51:37.000><c> of</c>\nhold um so the optimal number of\nhold um so the optimal number of parameters<00:51:37.720><c> that</c><00:51:38.000><c> that</c><00:51:38.119><c> chinchilla</c><00:51:38.640><c> Pap</c><00:51:39.000><c> have</c>\nparameters that that chinchilla Pap have\nparameters that that chinchilla Pap have found<00:51:39.640><c> is</c><00:51:39.760><c> to</c><00:51:40.000><c> use</c><00:51:40.839><c> 20</c><00:51:41.240><c> tokens</c><00:51:41.839><c> for</c><00:51:42.119><c> every</c>\nfound is to use 20 tokens for every\nfound is to use 20 tokens for every parameter<00:51:42.799><c> that</c><00:51:42.880><c> you</c><00:51:43.000><c> train</c><00:51:44.000><c> uh</c><00:51:44.079><c> so</c><00:51:44.240><c> if</c><00:51:44.319><c> you</c>\nparameter that you train uh so if you\nparameter that you train uh so if you add<00:51:44.640><c> one</c><00:51:44.760><c> more</c><00:51:44.920><c> parameter</c><00:51:45.440><c> you</c><00:51:45.520><c> should</c><00:51:45.799><c> add</c>\nadd one more parameter you should add\nadd one more parameter you should add you<00:51:46.000><c> should</c><00:51:46.200><c> train</c><00:51:46.440><c> your</c><00:51:46.720><c> thing</c><00:51:46.880><c> on</c><00:51:47.359><c> your</c>\nyou should train your thing on your\nyou should train your thing on your model<00:51:47.720><c> on</c><00:51:47.799><c> 20</c><00:51:48.040><c> more</c><00:51:48.640><c> tokens</c><00:51:49.640><c> so</c><00:51:49.880><c> one</c><00:51:50.280><c> caveat</c>\nmodel on 20 more tokens so one caveat\nmodel on 20 more tokens so one caveat here<00:51:51.000><c> is</c><00:51:51.119><c> that</c><00:51:51.280><c> this</c><00:51:51.400><c> is</c><00:51:51.599><c> optimal</c><00:51:52.000><c> training</c>\nhere is that this is optimal training\nhere is that this is optimal training resources<00:51:53.200><c> so</c><00:51:53.359><c> that</c><00:51:53.480><c> is</c><00:51:53.680><c> telling</c><00:51:53.960><c> me</c><00:51:54.280><c> if</c><00:51:54.400><c> you</c>\nresources so that is telling me if you\nresources so that is telling me if you have<00:51:55.240><c> 10^</c><00:51:55.880><c> 23</c><00:51:56.240><c> FL</c>\nhave 10^ 23 FL\nhave 10^ 23 FL or<00:51:57.359><c> if</c><00:51:57.440><c> you</c><00:51:57.559><c> have</c><00:51:57.720><c> like</c><00:51:57.960><c> 100</c><00:51:58.400><c> I</c><00:51:58.480><c> don't</c><00:51:58.599><c> know</c><00:51:58.799><c> how</c>\nor if you have like 100 I don't know how\nor if you have like 100 I don't know how much<00:51:59.119><c> that</c><00:51:59.240><c> is100</c><00:51:59.880><c> million</c><00:52:00.880><c> or</c><00:52:01.119><c> 10</c><00:52:01.400><c> no</c><00:52:01.720><c> that's</c>\nmuch that is100 million or 10 no that's\nmuch that is100 million or 10 no that's much<00:52:02.240><c> less</c><00:52:02.480><c> actually</c><00:52:02.799><c> let's</c><00:52:02.960><c> say</c><00:52:03.079><c> I</c><00:52:03.200><c> have</c><00:52:03.280><c> $5</c>\nmuch less actually let's say I have $5\nmuch less actually let's say I have $5 million<00:52:04.119><c> to</c><00:52:04.319><c> to</c><00:52:04.839><c> train</c><00:52:05.240><c> my</c><00:52:05.480><c> best</c><00:52:05.839><c> model</c><00:52:06.280><c> that</c>\nmillion to to train my best model that\nmillion to to train my best model that gets<00:52:06.599><c> the</c><00:52:06.720><c> lowest</c><00:52:07.040><c> loss</c><00:52:07.680><c> how</c><00:52:07.960><c> how</c><00:52:08.200><c> what</c><00:52:08.359><c> would</c>\ngets the lowest loss how how what would\ngets the lowest loss how how what would I<00:52:08.799><c> train</c><00:52:09.119><c> on</c><00:52:09.920><c> in</c><00:52:10.119><c> reality</c><00:52:10.599><c> these</c><00:52:10.799><c> companies</c>\nI train on in reality these companies\nI train on in reality these companies need<00:52:11.280><c> to</c><00:52:11.400><c> think</c><00:52:11.559><c> about</c><00:52:11.799><c> inference</c><00:52:12.240><c> also</c><00:52:12.920><c> if</c>\nneed to think about inference also if\nneed to think about inference also if you<00:52:13.160><c> have</c><00:52:13.240><c> a</c><00:52:13.359><c> smaller</c><00:52:13.760><c> model</c><00:52:14.760><c> they</c><00:52:14.920><c> will</c><00:52:15.559><c> spend</c>\nyou have a smaller model they will spend\nyou have a smaller model they will spend less<00:52:16.280><c> over</c><00:52:16.599><c> time</c><00:52:17.520><c> um</c><00:52:17.920><c> so</c><00:52:18.160><c> actually</c><00:52:18.640><c> if</c><00:52:18.720><c> you</c>\nless over time um so actually if you\nless over time um so actually if you consider<00:52:19.240><c> the</c><00:52:19.359><c> inference</c><00:52:19.799><c> cost</c><00:52:20.160><c> you</c><00:52:20.280><c> have</c>\nconsider the inference cost you have\nconsider the inference cost you have other<00:52:20.640><c> papers</c><00:52:20.920><c> that</c><00:52:21.040><c> Tred</c><00:52:21.240><c> to</c><00:52:21.359><c> show</c><00:52:21.640><c> that</c><00:52:22.319><c> um</c>\nother papers that Tred to show that um\nother papers that Tred to show that um it's<00:52:22.920><c> around</c>\nit's around\nit's around 150<00:52:25.000><c> uh</c><00:52:25.240><c> parameters</c><00:52:26.079><c> per</c><00:52:26.280><c> sorry</c><00:52:26.880><c> tokens</c><00:52:27.400><c> per</c>\n150 uh parameters per sorry tokens per\n150 uh parameters per sorry tokens per parameters<00:52:28.160><c> because</c><00:52:28.319><c> you</c><00:52:28.480><c> prefer</c><00:52:29.079><c> having</c><00:52:29.319><c> a</c>\nparameters because you prefer having a\nparameters because you prefer having a smaller<00:52:29.920><c> model</c><00:52:30.760><c> cuz</c><00:52:31.000><c> over</c><00:52:31.280><c> time</c><00:52:31.760><c> you're</c><00:52:31.920><c> going</c>\nsmaller model cuz over time you're going\nsmaller model cuz over time you're going to<00:52:32.440><c> you're</c><00:52:32.599><c> going</c><00:52:32.680><c> to</c><00:52:32.880><c> actually</c><00:52:33.839><c> um</c><00:52:34.720><c> spend</c>\nto you're going to actually um spend\nto you're going to actually um spend less<00:52:35.319><c> money</c><00:52:35.920><c> on</c><00:52:36.119><c> inference</c><00:52:36.520><c> of</c><00:52:36.680><c> these</c><00:52:36.839><c> models</c>\nless money on inference of these models\nless money on inference of these models so<00:52:37.880><c> 150</c><00:52:38.760><c> to</c><00:52:39.000><c> one</c><00:52:39.319><c> that's</c><00:52:39.720><c> around</c><00:52:40.240><c> what</c><00:52:40.480><c> the</c>\nso 150 to one that's around what the\nso 150 to one that's around what the best<00:52:40.920><c> models</c><00:52:41.599><c> are</c><00:52:41.880><c> trained</c><00:52:42.280><c> on</c><00:52:42.680><c> right</c><00:52:42.799><c> now</c><00:52:43.040><c> at</c>\nbest models are trained on right now at\nbest models are trained on right now at least<00:52:43.319><c> the</c><00:52:43.440><c> ones</c><00:52:43.680><c> that</c><00:52:43.839><c> are</c><00:52:44.760><c> that</c><00:52:44.880><c> are</c><00:52:45.040><c> used</c><00:52:45.720><c> um</c>\nleast the ones that are that are used um\nleast the ones that are that are used um in<00:52:46.760><c> practice</c><00:52:47.440><c> for</c><00:52:47.599><c> in</c>\nin practice for in\nin practice for in production\nproduction\nproduction great<00:52:52.000><c> any</c><00:52:52.160><c> question</c><00:52:52.400><c> on</c>\nchin<00:52:56.040><c> great</c><00:52:56.920><c> oh</c><00:52:57.160><c> sorry</c><00:52:57.799><c> in</c><00:52:58.079><c> practice</c><00:52:58.599><c> how</c>\nchin great oh sorry in practice how\nchin great oh sorry in practice how expensive<00:52:59.440><c> is</c><00:52:59.799><c> inference</c><00:53:00.400><c> for</c><00:53:00.599><c> these</c><00:53:00.799><c> models</c>\nexpensive is inference for these models\nexpensive is inference for these models rela<00:53:01.799><c> to</c>\nrela to\nrela to train<00:53:03.520><c> actually</c><00:53:03.839><c> very</c><00:53:04.160><c> expensive</c><00:53:05.160><c> uh</c><00:53:05.319><c> I</c><00:53:05.400><c> will</c>\ntrain actually very expensive uh I will\ntrain actually very expensive uh I will not<00:53:05.839><c> talk</c><00:53:06.040><c> about</c><00:53:06.240><c> inference</c><00:53:06.680><c> because</c><00:53:06.839><c> that</c>\nnot talk about inference because that\nnot talk about inference because that would<00:53:07.119><c> be</c><00:53:07.319><c> another</c><00:53:07.720><c> entire</c><00:53:08.520><c> lecture</c><00:53:09.119><c> but</c><00:53:09.760><c> just</c>\nwould be another entire lecture but just\nwould be another entire lecture but just think<00:53:10.160><c> about</c><00:53:10.520><c> Chad</c><00:53:10.799><c> GPT</c><00:53:11.400><c> where</c><00:53:11.559><c> they</c><00:53:11.799><c> have</c><00:53:12.480><c> I</c>\nthink about Chad GPT where they have I\nthink about Chad GPT where they have I don't<00:53:12.720><c> know</c><00:53:12.839><c> how</c><00:53:12.960><c> much</c><00:53:13.400><c> it</c><00:53:13.480><c> is</c><00:53:13.799><c> now</c><00:53:14.040><c> like</c><00:53:14.200><c> 600</c>\ndon't know how much it is now like 600\ndon't know how much it is now like 600 million<00:53:15.680><c> people</c><00:53:16.040><c> that</c><00:53:16.160><c> used</c><00:53:16.480><c> it</c><00:53:17.440><c> um</c><00:53:18.640><c> like</c>\nmillion people that used it um like\nmillion people that used it um like that's<00:53:20.000><c> a</c><00:53:20.319><c> lot</c>\nthat's a lot\nthat's a lot um<00:53:22.839><c> yeah</c><00:53:23.200><c> so</c><00:53:23.400><c> it's</c><00:53:23.599><c> actually</c><00:53:23.839><c> very</c><00:53:24.040><c> expensive</c>\num yeah so it's actually very expensive\num yeah so it's actually very expensive there's<00:53:24.720><c> a</c><00:53:24.880><c> lot</c><00:53:25.000><c> of</c><00:53:25.160><c> optimization</c><00:53:25.720><c> you</c><00:53:25.799><c> can</c><00:53:25.920><c> do</c>\nthere's a lot of optimization you can do\nthere's a lot of optimization you can do for<00:53:26.240><c> in</c><00:53:26.799><c> though</c><00:53:27.359><c> um</c><00:53:27.599><c> and</c><00:53:27.720><c> that's</c><00:53:27.880><c> an</c><00:53:28.079><c> entire</c>\nfor in though um and that's an entire\nfor in though um and that's an entire other<00:53:28.640><c> lecture</c><00:53:29.000><c> so</c><00:53:29.119><c> I'm</c><00:53:29.240><c> going</c><00:53:29.319><c> to</c><00:53:29.480><c> skip</c><00:53:29.799><c> that</c>\nother lecture so I'm going to skip that\nother lecture so I'm going to skip that uh<00:53:30.680><c> this</c><00:53:30.839><c> time</c><00:53:31.720><c> but</c><00:53:31.839><c> it's</c><00:53:32.000><c> very</c>\nuh this time but it's very\nuh this time but it's very interesting<00:53:33.839><c> okay</c><00:53:34.040><c> tuning</c><00:53:34.960><c> um</c><00:53:35.240><c> as</c><00:53:35.359><c> I</c><00:53:35.520><c> said</c>\ninteresting okay tuning um as I said\ninteresting okay tuning um as I said there<00:53:35.920><c> are</c><00:53:36.079><c> many</c><00:53:36.280><c> things</c><00:53:36.480><c> that</c><00:53:36.599><c> you</c><00:53:36.680><c> can</c><00:53:37.240><c> uh</c>\nthere are many things that you can uh\nthere are many things that you can uh answer<00:53:37.640><c> with</c><00:53:37.760><c> scaling</c><00:53:38.079><c> laws</c><00:53:38.400><c> I</c><00:53:38.480><c> just</c><00:53:38.640><c> try</c><00:53:38.839><c> to</c>\nanswer with scaling laws I just try to\nanswer with scaling laws I just try to give<00:53:39.160><c> you</c><00:53:39.720><c> two</c><00:53:40.079><c> examples</c><00:53:41.079><c> uh</c><00:53:41.200><c> but</c><00:53:41.319><c> really</c>\ngive you two examples uh but really\ngive you two examples uh but really there<00:53:41.640><c> are</c><00:53:41.760><c> many</c><00:53:41.960><c> things</c><00:53:42.319><c> what</c><00:53:42.559><c> data</c><00:53:42.760><c> do</c><00:53:42.880><c> you</c>\nthere are many things what data do you\nthere are many things what data do you use<00:53:43.400><c> what</c><00:53:43.559><c> mixture</c><00:53:44.280><c> what</c><00:53:44.640><c> data</c><00:53:44.920><c> mixing</c>\nuse what mixture what data mixing\nuse what mixture what data mixing waiting<00:53:46.200><c> you</c><00:53:46.359><c> use</c><00:53:46.720><c> data</c><00:53:46.920><c> mixtures</c><00:53:47.319><c> that's</c>\nwaiting you use data mixtures that's\nwaiting you use data mixtures that's what<00:53:47.559><c> we</c><00:53:47.680><c> talked</c><00:53:47.920><c> about</c><00:53:48.160><c> before</c><00:53:49.040><c> uh</c><00:53:49.200><c> what</c>\nwhat we talked about before uh what\nwhat we talked about before uh what architecture<00:53:49.839><c> you</c><00:53:50.000><c> use</c><00:53:50.599><c> whether</c><00:53:50.799><c> you</c><00:53:50.880><c> should</c>\narchitecture you use whether you should\narchitecture you use whether you should make<00:53:51.240><c> your</c><00:53:51.359><c> models</c><00:53:52.079><c> uh</c><00:53:52.280><c> wider</c><00:53:52.720><c> or</c><00:53:53.000><c> deeper</c><00:53:54.000><c> um</c>\nmake your models uh wider or deeper um\nmake your models uh wider or deeper um should<00:53:54.400><c> you</c><00:53:54.559><c> be</c><00:53:55.200><c> paying</c><00:53:55.440><c> for</c><00:53:55.599><c> more</c><00:53:55.799><c> gpus</c><00:53:56.240><c> or</c>\nshould you be paying for more gpus or\nshould you be paying for more gpus or actually<00:53:56.720><c> collecting</c><00:53:57.079><c> more</c><00:53:57.319><c> data</c><00:53:58.280><c> um</c><00:53:59.000><c> all</c>\nactually collecting more data um all\nactually collecting more data um all these<00:53:59.359><c> things</c><00:53:59.599><c> are</c><00:53:59.799><c> things</c><00:54:00.000><c> you</c><00:54:00.119><c> can</c><00:54:00.240><c> try</c><00:54:00.440><c> to</c>\nthese things are things you can try to\nthese things are things you can try to answer<00:54:00.839><c> with</c><00:54:00.960><c> scaling</c>\nanswer with scaling\nanswer with scaling laws<00:54:03.400><c> one</c><00:54:03.640><c> thing</c><00:54:03.799><c> I</c><00:54:03.880><c> want</c><00:54:04.000><c> to</c><00:54:04.160><c> say</c><00:54:04.520><c> is</c><00:54:04.680><c> the</c><00:54:04.799><c> bit</c>\nlaws one thing I want to say is the bit\nlaws one thing I want to say is the bit lesson<00:54:05.559><c> if</c><00:54:05.640><c> you</c><00:54:05.760><c> ever</c><00:54:06.040><c> heard</c><00:54:06.720><c> of</c><00:54:06.960><c> Richard</c>\nlesson if you ever heard of Richard\nlesson if you ever heard of Richard sudden<00:54:08.160><c> a</c><00:54:08.359><c> very</c><00:54:08.599><c> famous</c><00:54:08.920><c> blog</c><00:54:09.200><c> post</c><00:54:09.400><c> in</c><00:54:09.880><c> 2019</c>\nsudden a very famous blog post in 2019\nsudden a very famous blog post in 2019 um<00:54:11.400><c> what</c><00:54:11.559><c> he</c><00:54:11.920><c> realized</c><00:54:12.920><c> uh</c><00:54:13.640><c> which</c><00:54:14.640><c> I</c><00:54:14.799><c> think</c><00:54:15.520><c> not</c>\num what he realized uh which I think not\num what he realized uh which I think not enough<00:54:16.000><c> people</c><00:54:16.200><c> realize</c><00:54:16.599><c> I</c><00:54:16.760><c> didn't</c>\nenough people realize I didn't\nenough people realize I didn't definitely<00:54:17.480><c> did</c><00:54:17.640><c> not</c><00:54:17.799><c> realize</c><00:54:18.160><c> at</c><00:54:18.359><c> that</c><00:54:18.520><c> time</c>\ndefinitely did not realize at that time\ndefinitely did not realize at that time um<00:54:20.040><c> is</c><00:54:20.160><c> that</c><00:54:20.680><c> once</c><00:54:20.839><c> you</c><00:54:20.960><c> see</c><00:54:21.240><c> these</c><00:54:21.400><c> type</c><00:54:21.559><c> of</c>\num is that once you see these type of\num is that once you see these type of scaling<00:54:21.960><c> laws</c><00:54:22.440><c> you</c><00:54:22.599><c> know</c><00:54:22.839><c> that</c><00:54:23.000><c> the</c><00:54:23.119><c> more</c>\nscaling laws you know that the more\nscaling laws you know that the more compute<00:54:23.720><c> you</c><00:54:23.920><c> have</c><00:54:24.319><c> the</c><00:54:24.440><c> better</c><00:54:24.720><c> models</c><00:54:25.440><c> you</c>\ncompute you have the better models you\ncompute you have the better models you will<00:54:25.799><c> get</c><00:54:26.119><c> so</c><00:54:26.480><c> with</c><00:54:26.599><c> skill</c><00:54:26.839><c> you</c><00:54:26.920><c> will</c><00:54:27.079><c> get</c>\nwill get so with skill you will get\nwill get so with skill you will get better<00:54:27.480><c> model</c><00:54:28.119><c> and</c><00:54:28.200><c> you</c><00:54:28.359><c> also</c><00:54:28.599><c> know</c><00:54:28.839><c> by</c><00:54:29.119><c> Mo</c><00:54:29.559><c> law</c>\nbetter model and you also know by Mo law\nbetter model and you also know by Mo law or<00:54:30.359><c> these</c><00:54:30.559><c> type</c><00:54:30.760><c> of</c><00:54:30.960><c> variant</c><00:54:31.280><c> of</c><00:54:31.440><c> Mo</c><00:54:31.760><c> law</c><00:54:32.200><c> that</c>\nor these type of variant of Mo law that\nor these type of variant of Mo law that you<00:54:32.440><c> will</c><00:54:32.720><c> always</c><00:54:33.000><c> have</c><00:54:33.200><c> better</c><00:54:33.440><c> compute</c><00:54:34.079><c> then</c>\nyou will always have better compute then\nyou will always have better compute then the<00:54:34.480><c> only</c><00:54:34.799><c> thing</c><00:54:35.400><c> that</c><00:54:35.640><c> matters</c><00:54:36.400><c> is</c><00:54:36.599><c> just</c><00:54:36.760><c> to</c>\nthe only thing that matters is just to\nthe only thing that matters is just to have<00:54:37.359><c> architectures</c><00:54:38.079><c> that</c><00:54:38.200><c> can</c><00:54:38.400><c> leverage</c>\nhave architectures that can leverage\nhave architectures that can leverage computation<00:54:39.920><c> so</c><00:54:40.160><c> what</c><00:54:40.319><c> matters</c><00:54:41.119><c> is</c><00:54:41.319><c> basically</c>\ncomputation so what matters is basically\ncomputation so what matters is basically systems<00:54:42.799><c> data</c><00:54:43.559><c> and</c><00:54:43.760><c> less</c><00:54:44.000><c> so</c><00:54:44.280><c> the</c>\nsystems data and less so the\nsystems data and less so the architecture<00:54:45.079><c> like</c><00:54:45.200><c> the</c><00:54:45.319><c> small</c><00:54:45.640><c> architecture</c>\narchitecture like the small architecture\narchitecture like the small architecture differences<00:54:46.640><c> like</c><00:54:46.880><c> your</c><00:54:47.280><c> your</c><00:54:47.680><c> your</c>\ndifferences like your your your\ndifferences like your your your activation<00:54:48.319><c> and</c><00:54:48.480><c> things</c><00:54:48.680><c> like</c><00:54:48.880><c> this</c><00:54:49.640><c> uh</c><00:54:49.799><c> so</c><00:54:49.960><c> I</c>\nactivation and things like this uh so I\nactivation and things like this uh so I think<00:54:50.200><c> that's</c><00:54:50.400><c> like</c><00:54:50.559><c> one</c><00:54:50.680><c> of</c><00:54:50.799><c> the</c><00:54:50.920><c> reasons</c><00:54:51.280><c> why</c>\nthink that's like one of the reasons why\nthink that's like one of the reasons why most<00:54:51.640><c> of</c><00:54:51.839><c> research</c><00:54:52.200><c> focuses</c><00:54:53.040><c> on</c><00:54:53.559><c> um</c><00:54:54.440><c> some</c>\nmost of research focuses on um some\nmost of research focuses on um some things<00:54:54.960><c> that</c><00:54:55.119><c> for</c><00:54:55.400><c> industry</c><00:54:55.720><c> matters</c><00:54:56.079><c> less</c>\nthings that for industry matters less\nthings that for industry matters less and<00:54:56.960><c> I</c><00:54:57.119><c> was</c><00:54:57.280><c> one</c><00:54:57.400><c> of</c><00:54:57.559><c> those</c><00:54:57.760><c> researchers</c><00:54:58.280><c> for</c><00:54:58.799><c> a</c>\nand I was one of those researchers for a\nand I was one of those researchers for a large<00:54:59.680><c> part</c><00:54:59.839><c> of</c><00:55:00.040><c> my</c><00:55:00.319><c> my</c><00:55:00.880><c> career</c><00:55:01.880><c> um</c><00:55:02.520><c> so</c><00:55:02.720><c> don't</c>\nlarge part of my my career um so don't\nlarge part of my my career um so don't spend<00:55:03.200><c> time</c><00:55:03.480><c> over</c><00:55:03.880><c> complicating</c><00:55:04.880><c> do</c><00:55:05.200><c> the</c>\nspend time over complicating do the\nspend time over complicating do the simple<00:55:05.720><c> things</c><00:55:06.280><c> do</c><00:55:06.440><c> it</c><00:55:06.640><c> well</c><00:55:07.040><c> seal</c><00:55:07.559><c> them</c>\nsimple things do it well seal them\nsimple things do it well seal them that's<00:55:08.359><c> really</c><00:55:08.640><c> what</c><00:55:08.920><c> openi</c><00:55:09.520><c> taught</c><00:55:09.760><c> us</c><00:55:10.359><c> with</c>\nthat's really what openi taught us with\nthat's really what openi taught us with um<00:55:11.119><c> with</c><00:55:11.280><c> chat</c><00:55:11.480><c> gpg</c><00:55:12.079><c> and</c><00:55:12.240><c> with</c><00:55:12.440><c> all</c><00:55:12.559><c> the</c><00:55:12.680><c> gpts</c>\num with chat gpg and with all the gpts\num with chat gpg and with all the gpts before<00:55:15.480><c> okay</c><00:55:15.640><c> I</c><00:55:15.720><c> want</c><00:55:15.839><c> to</c><00:55:16.000><c> give</c><00:55:16.119><c> you</c><00:55:16.319><c> some</c>\nbefore okay I want to give you some\nbefore okay I want to give you some backup<00:55:17.000><c> the</c><00:55:17.200><c> envelope</c><00:55:18.200><c> computation</c><00:55:18.799><c> so</c><00:55:18.960><c> I</c>\nbackup the envelope computation so I\nbackup the envelope computation so I might<00:55:19.200><c> be</c><00:55:19.400><c> off</c><00:55:19.720><c> by</c><00:55:19.839><c> a</c><00:55:19.960><c> few</c><00:55:20.119><c> factors</c><00:55:20.559><c> here</c><00:55:20.720><c> but</c><00:55:20.839><c> I</c>\nmight be off by a few factors here but I\nmight be off by a few factors here but I just<00:55:21.000><c> want</c><00:55:21.119><c> to</c><00:55:21.280><c> give</c><00:55:21.400><c> you</c><00:55:21.559><c> a</c><00:55:21.760><c> sense</c><00:55:22.079><c> of</c><00:55:22.319><c> how</c>\njust want to give you a sense of how\njust want to give you a sense of how costly<00:55:23.319><c> it</c><00:55:23.440><c> is</c><00:55:23.559><c> to</c><00:55:23.680><c> train</c><00:55:23.960><c> some</c><00:55:24.079><c> of</c><00:55:24.200><c> these</c>\ncostly it is to train some of these\ncostly it is to train some of these models<00:55:25.280><c> I'll</c><00:55:25.480><c> give</c><00:55:25.640><c> as</c><00:55:25.760><c> an</c><00:55:25.920><c> example</c>\nmodels I'll give as an example\nmodels I'll give as an example Lama<00:55:27.240><c> 3</c><00:55:27.520><c> 400b</c><00:55:28.280><c> which</c><00:55:28.359><c> is</c><00:55:28.559><c> currently</c><00:55:28.960><c> the</c><00:55:29.079><c> best</c>\nLama 3 400b which is currently the best\nLama 3 400b which is currently the best open<00:55:29.559><c> source</c><00:55:29.839><c> model</c><00:55:30.119><c> that</c><00:55:30.240><c> you</c><00:55:30.319><c> can</c><00:55:30.640><c> get</c><00:55:31.640><c> uh</c><00:55:31.880><c> it</c>\nopen source model that you can get uh it\nopen source model that you can get uh it was<00:55:32.240><c> trained</c><00:55:32.680><c> on</c><00:55:33.240><c> 15.6</c><00:55:34.079><c> tokens</c><00:55:35.039><c> it</c><00:55:35.240><c> has</c><00:55:35.760><c> 45</c>\nwas trained on 15.6 tokens it has 45\nwas trained on 15.6 tokens it has 45 billion<00:55:37.119><c> parameters</c><00:55:37.839><c> so</c><00:55:38.160><c> just</c><00:55:38.440><c> now</c><00:55:38.599><c> that</c><00:55:38.720><c> you</c>\nbillion parameters so just now that you\nbillion parameters so just now that you know<00:55:39.119><c> what</c><00:55:39.240><c> is</c><00:55:39.440><c> like</c><00:55:39.680><c> this</c><00:55:40.119><c> uh</c><00:55:40.559><c> optimal</c><00:55:41.119><c> tokens</c>\nknow what is like this uh optimal tokens\nknow what is like this uh optimal tokens per<00:55:41.680><c> parameter</c><00:55:42.160><c> that's</c><00:55:42.400><c> around</c><00:55:42.599><c> 40</c><00:55:43.200><c> so</c><00:55:43.400><c> that's</c>\nper parameter that's around 40 so that's\nper parameter that's around 40 so that's a<00:55:44.000><c> little</c><00:55:44.160><c> bit</c><00:55:44.319><c> more</c><00:55:44.480><c> than</c><00:55:44.640><c> chinchilla</c><00:55:45.480><c> but</c>\na little bit more than chinchilla but\na little bit more than chinchilla but less<00:55:45.960><c> than</c><00:55:46.160><c> this</c><00:55:46.400><c> like</c><00:55:46.720><c> inference</c><00:55:47.720><c> uh</c><00:55:48.039><c> optimal</c>\nless than this like inference uh optimal\nless than this like inference uh optimal um<00:55:50.039><c> model</c><00:55:50.440><c> so</c><00:55:50.599><c> they</c><00:55:50.720><c> went</c><00:55:50.960><c> for</c><00:55:51.119><c> training</c>\num model so they went for training\num model so they went for training optimality<00:55:53.400><c> uh</c><00:55:53.520><c> flops</c><00:55:54.119><c> for</c><00:55:54.359><c> this</c><00:55:54.559><c> model</c><00:55:55.000><c> so</c>\noptimality uh flops for this model so\noptimality uh flops for this model so one<00:55:55.680><c> simple</c><00:55:56.559><c> uh</c><00:55:56.680><c> way</c><00:55:56.839><c> to</c><00:55:57.000><c> compute</c><00:55:57.319><c> flops</c><00:55:57.720><c> is</c>\none simple uh way to compute flops is\none simple uh way to compute flops is six<00:55:58.960><c> uh</c><00:55:59.240><c> times</c><00:55:59.720><c> the</c><00:55:59.839><c> number</c><00:56:00.079><c> of</c><00:56:00.240><c> parameters</c>\nsix uh times the number of parameters\nsix uh times the number of parameters times<00:56:01.160><c> the</c><00:56:01.240><c> number</c><00:56:01.440><c> of</c><00:56:01.520><c> data</c><00:56:01.839><c> you</c><00:56:01.960><c> train</c><00:56:02.200><c> on</c><00:56:02.880><c> uh</c>\ntimes the number of data you train on uh\ntimes the number of data you train on uh so<00:56:03.119><c> if</c><00:56:03.200><c> you</c><00:56:03.280><c> do</c><00:56:03.400><c> the</c><00:56:03.520><c> simple</c><00:56:03.839><c> calculation</c><00:56:04.440><c> here</c>\nso if you do the simple calculation here\nso if you do the simple calculation here it's<00:56:05.000><c> 3.8</c><00:56:05.799><c> e25</c><00:56:06.720><c> flops</c><00:56:07.720><c> the</c><00:56:07.839><c> reason</c><00:56:08.160><c> why</c><00:56:08.319><c> this</c>\nit's 3.8 e25 flops the reason why this\nit's 3.8 e25 flops the reason why this is<00:56:08.599><c> important</c><00:56:09.119><c> is</c><00:56:09.240><c> that</c><00:56:09.440><c> if</c><00:56:09.559><c> you</c><00:56:09.680><c> follow</c><00:56:10.039><c> the</c>\nis important is that if you follow the\nis important is that if you follow the little<00:56:10.280><c> bit</c><00:56:10.440><c> the</c><00:56:10.520><c> news</c><00:56:10.760><c> there's</c><00:56:10.920><c> an</c><00:56:11.079><c> executive</c>\nlittle bit the news there's an executive\nlittle bit the news there's an executive order<00:56:12.319><c> from</c><00:56:12.520><c> Biden</c><00:56:12.920><c> that</c><00:56:13.119><c> basically</c><00:56:13.440><c> says</c>\norder from Biden that basically says\norder from Biden that basically says that<00:56:13.799><c> once</c><00:56:13.960><c> you</c><00:56:14.079><c> have</c><00:56:14.880><c> uh</c><00:56:15.000><c> 1</c><00:56:15.799><c> e26</c><00:56:16.799><c> parameters</c>\nthat once you have uh 1 e26 parameters\nthat once you have uh 1 e26 parameters uh<00:56:17.920><c> sorry</c><00:56:18.240><c> flops</c><00:56:19.240><c> uh</c><00:56:19.359><c> then</c><00:56:19.520><c> you</c><00:56:19.640><c> have</c><00:56:19.799><c> special</c>\nuh sorry flops uh then you have special\nuh sorry flops uh then you have special scrutiny<00:56:20.559><c> on</c><00:56:20.680><c> your</c><00:56:20.799><c> models</c><00:56:21.359><c> so</c><00:56:21.599><c> they</c><00:56:21.760><c> went</c><00:56:22.319><c> 2x</c>\nscrutiny on your models so they went 2x\nscrutiny on your models so they went 2x less<00:56:23.160><c> than</c><00:56:23.359><c> that</c><00:56:23.520><c> so</c><00:56:23.720><c> they</c><00:56:23.920><c> really</c><00:56:24.119><c> went</c><00:56:24.440><c> right</c>\nless than that so they really went right\nless than that so they really went right below<00:56:25.000><c> this</c><00:56:25.480><c> to</c><00:56:25.640><c> not</c><00:56:25.839><c> have</c><00:56:25.960><c> special</c><00:56:26.440><c> scrutiny</c>\nbelow this to not have special scrutiny\nbelow this to not have special scrutiny so<00:56:27.559><c> 38</c><00:56:28.559><c> uh</c><00:56:28.680><c> I</c><00:56:28.799><c> might</c><00:56:28.960><c> be</c><00:56:29.119><c> off</c><00:56:29.319><c> by</c><00:56:29.480><c> a</c><00:56:29.599><c> little</c><00:56:29.760><c> bit</c>\nso 38 uh I might be off by a little bit\nso 38 uh I might be off by a little bit but<00:56:30.039><c> it's</c><00:56:30.200><c> definitely</c><00:56:30.680><c> under</c><00:56:31.000><c> the</c><00:56:31.440><c> 1</c>\n26<00:56:35.520><c> oh</c><00:56:36.079><c> um</c><00:56:36.640><c> so</c><00:56:37.200><c> paramet</c><00:56:37.720><c> p</c><00:56:37.920><c> is</c><00:56:38.079><c> parameters</c><00:56:39.000><c> n</c><00:56:39.720><c> is</c>\n26 oh um so paramet p is parameters n is\n26 oh um so paramet p is parameters n is data<00:56:40.559><c> number</c><00:56:40.799><c> of</c><00:56:40.960><c> tokens</c><00:56:41.880><c> this</c><00:56:42.039><c> is</c><00:56:42.400><c> a</c><00:56:43.280><c> uh</c><00:56:43.599><c> this</c>\ndata number of tokens this is a uh this\ndata number of tokens this is a uh this is<00:56:43.799><c> just</c><00:56:43.920><c> an</c>\nis just an\nis just an approximation<00:56:45.920><c> we</c>\napproximation we\napproximation we yeah<00:56:48.280><c> okay</c><00:56:48.880><c> uh</c><00:56:49.079><c> compute</c><00:56:49.960><c> and</c><00:56:50.480><c> we</c><00:56:50.599><c> know</c><00:56:50.880><c> that</c>\nyeah okay uh compute and we know that\nyeah okay uh compute and we know that they<00:56:51.160><c> trained</c><00:56:51.520><c> on</c><00:56:51.799><c> 16,000</c>\nthey trained on 16,000\nthey trained on 16,000 h100s<00:56:54.319><c> um</c><00:56:55.319><c> and</c><00:56:55.480><c> we</c><00:56:55.599><c> know</c><00:56:55.720><c> the</c><00:56:55.839><c> throughput</c><00:56:56.280><c> but</c>\nh100s um and we know the throughput but\nh100s um and we know the throughput but they<00:56:56.880><c> they</c><00:56:56.960><c> said</c><00:56:57.200><c> it</c><00:56:57.400><c> too</c><00:56:58.400><c> uh</c><00:56:58.520><c> so</c><00:56:58.760><c> if</c><00:56:58.880><c> you</c><00:56:58.960><c> do</c>\nthey they said it too uh so if you do\nthey they said it too uh so if you do the<00:56:59.280><c> computation</c><00:57:00.200><c> it</c><00:57:00.359><c> takes</c><00:57:00.640><c> around</c><00:57:00.960><c> 70</c><00:57:01.480><c> days</c>\nthe computation it takes around 70 days\nthe computation it takes around 70 days um<00:57:02.839><c> or</c><00:57:03.079><c> 26</c><00:57:03.640><c> million</c><00:57:04.039><c> GPU</c><00:57:04.640><c> hours</c><00:57:05.480><c> at</c><00:57:05.599><c> least</c>\num or 26 million GPU hours at least\num or 26 million GPU hours at least that's<00:57:05.960><c> with</c><00:57:06.200><c> my</c><00:57:06.839><c> uh</c><00:57:07.039><c> back</c><00:57:07.200><c> of</c><00:57:07.359><c> the</c><00:57:07.480><c> envelope</c>\nthat's with my uh back of the envelope\nthat's with my uh back of the envelope computation<00:57:08.480><c> they</c><00:57:08.640><c> actually</c><00:57:08.839><c> said</c><00:57:09.079><c> that</c><00:57:09.200><c> they</c>\ncomputation they actually said that they\ncomputation they actually said that they use<00:57:09.799><c> 30</c><00:57:10.200><c> million</c><00:57:10.680><c> instead</c><00:57:10.920><c> of</c><00:57:11.319><c> 26</c><00:57:11.760><c> million</c><00:57:12.079><c> GPU</c>\nuse 30 million instead of 26 million GPU\nuse 30 million instead of 26 million GPU hours<00:57:13.559><c> um</c><00:57:14.000><c> so</c><00:57:14.240><c> maybe</c><00:57:14.480><c> they</c><00:57:14.640><c> had</c><00:57:14.880><c> like</c><00:57:15.520><c> some</c><00:57:16.520><c> uh</c>\nhours um so maybe they had like some uh\nhours um so maybe they had like some uh some<00:57:16.880><c> challenges</c><00:57:17.599><c> I</c><00:57:17.680><c> don't</c><00:57:17.880><c> really</c><00:57:18.039><c> know</c><00:57:18.319><c> but</c>\nsome challenges I don't really know but\nsome challenges I don't really know but if<00:57:18.720><c> you</c><00:57:18.880><c> follow</c><00:57:19.280><c> the</c><00:57:19.440><c> simple</c><00:57:19.760><c> computation</c>\nif you follow the simple computation\nif you follow the simple computation it's<00:57:20.440><c> around</c><00:57:20.680><c> 70</c><00:57:21.480><c> days</c><00:57:22.480><c> um</c><00:57:23.240><c> cost</c><00:57:24.240><c> uh</c><00:57:24.400><c> I</c><00:57:24.480><c> mean</c>\nit's around 70 days um cost uh I mean\nit's around 70 days um cost uh I mean this<00:57:25.000><c> it's</c><00:57:25.280><c> hard</c><00:57:25.520><c> to</c><00:57:26.280><c> to</c><00:57:26.520><c> approximate</c><00:57:27.079><c> but</c><00:57:27.240><c> I'm</c>\nthis it's hard to to approximate but I'm\nthis it's hard to to approximate but I'm just<00:57:27.480><c> going</c><00:57:27.559><c> to</c><00:57:27.720><c> say</c><00:57:27.920><c> it's</c><00:57:28.440><c> kind</c><00:57:28.559><c> of</c><00:57:28.760><c> the</c><00:57:28.960><c> rent</c>\njust going to say it's kind of the rent\njust going to say it's kind of the rent like<00:57:29.640><c> what</c><00:57:29.760><c> if</c><00:57:29.920><c> I</c><00:57:30.000><c> were</c><00:57:30.280><c> to</c><00:57:30.480><c> rent</c><00:57:31.280><c> h100s</c><00:57:32.280><c> that</c>\nlike what if I were to rent h100s that\nlike what if I were to rent h100s that many<00:57:32.920><c> h100s</c><00:57:33.799><c> for</c><00:57:34.400><c> that</c><00:57:34.559><c> many</c><00:57:34.880><c> days</c><00:57:35.160><c> how</c><00:57:35.280><c> much</c>\nmany h100s for that many days how much\nmany h100s for that many days how much will<00:57:35.599><c> I</c><00:57:35.839><c> pay</c><00:57:36.480><c> uh</c><00:57:36.599><c> h100</c><00:57:37.280><c> a</c><00:57:37.400><c> lower</c><00:57:37.720><c> bound</c><00:57:38.039><c> on</c><00:57:38.160><c> the</c>\nwill I pay uh h100 a lower bound on the\nwill I pay uh h100 a lower bound on the on<00:57:38.720><c> the</c><00:57:38.880><c> renting</c><00:57:39.880><c> uh</c><00:57:40.039><c> cost</c><00:57:40.280><c> of</c><00:57:40.480><c> h100</c><00:57:41.079><c> is</c><00:57:41.240><c> around</c>\non the renting uh cost of h100 is around\non the renting uh cost of h100 is around 2<00:57:41.760><c> hours</c><00:57:42.440><c> uh</c><00:57:42.520><c> $2</c><00:57:43.079><c> per</c><00:57:43.240><c> hour</c><00:57:44.000><c> so</c><00:57:44.160><c> if</c><00:57:44.240><c> you</c>\n2 hours uh $2 per hour so if you\n2 hours uh $2 per hour so if you multiply<00:57:44.839><c> this</c><00:57:45.000><c> by</c><00:57:45.160><c> 26</c><00:57:46.000><c> million</c><00:57:46.960><c> uh</c><00:57:47.160><c> hours</c><00:57:48.160><c> uh</c>\nmultiply this by 26 million uh hours uh\nmultiply this by 26 million uh hours uh you<00:57:48.480><c> get</c><00:57:48.760><c> 52</c><00:57:49.359><c> million</c><00:57:50.280><c> uh</c><00:57:50.440><c> dollars</c><00:57:51.000><c> so</c><00:57:51.240><c> they</c>\nyou get 52 million uh dollars so they\nyou get 52 million uh dollars so they probably<00:57:51.760><c> pay</c><00:57:52.079><c> less</c><00:57:52.280><c> than</c><00:57:52.520><c> that</c><00:57:53.200><c> but</c><00:57:53.599><c> not</c>\nprobably pay less than that but not\nprobably pay less than that but not actually<00:57:54.400><c> much</c><00:57:54.760><c> less</c><00:57:55.079><c> because</c><00:57:55.480><c> all</c><00:57:55.680><c> these</c><00:57:56.319><c> um</c>\nactually much less because all these um\nactually much less because all these um all<00:57:57.440><c> these</c><00:57:57.599><c> services</c><00:57:58.039><c> that</c><00:57:58.240><c> actually</c><00:57:58.440><c> rent</c>\nall these services that actually rent\nall these services that actually rent gpus<00:57:59.160><c> they</c><00:57:59.280><c> don't</c><00:57:59.480><c> make</c><00:57:59.720><c> that</c><00:57:59.839><c> much</c><00:58:00.000><c> money</c><00:58:00.520><c> so</c>\ngpus they don't make that much money so\ngpus they don't make that much money so it's<00:58:01.119><c> it's</c><00:58:01.480><c> probably</c><00:58:01.760><c> slightly</c><00:58:02.160><c> less</c><00:58:02.319><c> but</c><00:58:02.440><c> not</c>\nit's it's probably slightly less but not\nit's it's probably slightly less but not that<00:58:02.760><c> much</c><00:58:02.960><c> less</c><00:58:03.880><c> um</c><00:58:04.280><c> now</c><00:58:04.720><c> salary</c><00:58:05.720><c> I</c><00:58:05.839><c> said</c><00:58:06.160><c> 50</c>\nthat much less um now salary I said 50\nthat much less um now salary I said 50 employees<00:58:07.720><c> 500k</c><00:58:08.440><c> per</c>\nemployees 500k per\nemployees 500k per year<00:58:10.599><c> say</c><00:58:10.839><c> yeah</c><00:58:10.920><c> it's</c><00:58:11.039><c> probably</c><00:58:11.240><c> the</c><00:58:11.359><c> right</c>\nyear say yeah it's probably the right\nyear say yeah it's probably the right ballpark<00:58:12.200><c> 25</c><00:58:12.680><c> million</c><00:58:13.359><c> uh</c><00:58:13.440><c> so</c><00:58:13.559><c> if</c><00:58:13.640><c> you</c><00:58:13.760><c> put</c><00:58:13.960><c> all</c>\nballpark 25 million uh so if you put all\nballpark 25 million uh so if you put all together<00:58:14.640><c> around</c><00:58:14.960><c> 75</c><00:58:15.760><c> million</c><00:58:16.760><c> um</c><00:58:17.200><c> dollars</c>\ntogether around 75 million um dollars\ntogether around 75 million um dollars for\nfor\nfor training<00:58:19.240><c> uh</c><00:58:19.520><c> this</c><00:58:19.680><c> Slammer</c><00:58:20.200><c> model</c><00:58:21.079><c> I'm</c>\ntraining uh this Slammer model I'm\ntraining uh this Slammer model I'm probably<00:58:21.480><c> off</c><00:58:21.680><c> by</c><00:58:21.880><c> like</c><00:58:22.000><c> 10</c><00:58:22.200><c> million</c><00:58:22.640><c> but</c><00:58:23.079><c> but</c>\nprobably off by like 10 million but but\nprobably off by like 10 million but but that's<00:58:23.520><c> kind</c><00:58:23.640><c> of</c><00:58:23.920><c> right</c><00:58:24.520><c> uh</c><00:58:24.760><c> bpk</c>\nthat's kind of right uh bpk\nthat's kind of right uh bpk carbon<00:58:27.920><c> emitted</c><00:58:28.920><c> um</c><00:58:29.240><c> a</c><00:58:29.319><c> lot</c><00:58:29.480><c> of</c><00:58:29.640><c> people</c><00:58:30.000><c> might</c>\ncarbon emitted um a lot of people might\ncarbon emitted um a lot of people might ask<00:58:30.799><c> like</c><00:58:31.280><c> also</c><00:58:31.640><c> the</c><00:58:31.760><c> cost</c><00:58:32.000><c> is</c><00:58:32.119><c> not</c><00:58:32.240><c> the</c><00:58:32.319><c> only</c>\nask like also the cost is not the only\nask like also the cost is not the only thing<00:58:32.640><c> that</c><00:58:32.760><c> is</c><00:58:32.920><c> important</c><00:58:33.480><c> so</c><00:58:33.680><c> I</c><00:58:33.799><c> did</c><00:58:33.960><c> the</c>\nthing that is important so I did the\nthing that is important so I did the computation<00:58:35.319><c> um</c><00:58:35.920><c> it's</c><00:58:36.200><c> around</c><00:58:37.440><c> 4</c><00:58:38.440><c> uh</c><00:58:39.039><c> 4,000</c><00:58:40.039><c> um</c>\ncomputation um it's around 4 uh 4,000 um\ncomputation um it's around 4 uh 4,000 um tons<00:58:40.920><c> of</c><00:58:41.119><c> CO2</c><00:58:42.079><c> equivalent</c><00:58:43.079><c> that</c><00:58:43.240><c> is</c><00:58:43.440><c> actually</c>\ntons of CO2 equivalent that is actually\ntons of CO2 equivalent that is actually only<00:58:44.039><c> 2,000</c><00:58:44.839><c> return</c><00:58:45.119><c> tickets</c><00:58:45.440><c> from</c><00:58:45.599><c> JFK</c><00:58:46.200><c> to</c><00:58:46.760><c> uh</c>\nonly 2,000 return tickets from JFK to uh\nonly 2,000 return tickets from JFK to uh London<00:58:47.760><c> so</c><00:58:48.160><c> right</c><00:58:48.359><c> now</c><00:58:49.119><c> uh</c><00:58:49.319><c> carbon</c><00:58:49.640><c> emitted</c><00:58:50.000><c> is</c>\nLondon so right now uh carbon emitted is\nLondon so right now uh carbon emitted is actually<00:58:50.799><c> not</c><00:58:51.799><c> uh</c><00:58:51.920><c> I</c><00:58:51.960><c> mean</c><00:58:52.119><c> it's</c><00:58:52.359><c> huge</c><00:58:52.680><c> but</c>\nactually not uh I mean it's huge but\nactually not uh I mean it's huge but it's<00:58:53.039><c> not</c><00:58:53.359><c> like</c><00:58:53.720><c> um</c><00:58:54.880><c> meaningful</c><00:58:55.880><c> yeah</c><00:58:56.160><c> yet</c><00:58:56.760><c> I</c>\nit's not like um meaningful yeah yet I\nit's not like um meaningful yeah yet I think<00:58:57.520><c> in</c><00:58:58.200><c> maybe</c><00:58:58.760><c> GPT</c><00:58:59.280><c> 6</c><00:58:59.720><c> gpt7</c><00:59:00.720><c> once</c><00:59:00.920><c> you</c>\nthink in maybe GPT 6 gpt7 once you\nthink in maybe GPT 6 gpt7 once you multiply<00:59:01.559><c> this</c><00:59:01.720><c> by</c><00:59:02.039><c> 100</c><00:59:02.799><c> that</c><00:59:02.960><c> might</c><00:59:03.160><c> become</c><00:59:03.400><c> a</c>\nmultiply this by 100 that might become a\nmultiply this by 100 that might become a real<00:59:03.760><c> issue</c><00:59:04.359><c> right</c><00:59:04.559><c> now</c><00:59:04.720><c> it's</c><00:59:04.920><c> still</c><00:59:05.200><c> not</c><00:59:05.720><c> uh</c><00:59:05.960><c> I</c>\nreal issue right now it's still not uh I\nreal issue right now it's still not uh I think<00:59:06.920><c> um</c><00:59:07.200><c> an</c><00:59:07.359><c> issue</c><00:59:07.559><c> in</c><00:59:07.640><c> the</c><00:59:07.760><c> grand</c><00:59:07.960><c> scheme</c><00:59:08.200><c> of</c>\nthink um an issue in the grand scheme of\nthink um an issue in the grand scheme of things<00:59:09.640><c> next</c><00:59:09.920><c> model</c><00:59:10.319><c> the</c><00:59:10.440><c> way</c><00:59:10.559><c> you</c><00:59:10.640><c> should</c><00:59:11.000><c> be</c>\nthings next model the way you should be\nthings next model the way you should be thinking<00:59:11.440><c> about</c><00:59:11.640><c> these</c><00:59:11.799><c> models</c><00:59:12.440><c> is</c><00:59:12.559><c> that</c>\nthinking about these models is that\nthinking about these models is that every<00:59:13.079><c> new</c><00:59:13.319><c> generation</c><00:59:14.079><c> the</c><00:59:14.200><c> number</c><00:59:14.440><c> of</c><00:59:14.559><c> flops</c>\nevery new generation the number of flops\nevery new generation the number of flops essentially<00:59:16.039><c> uh</c><00:59:16.160><c> multiplies</c><00:59:16.680><c> 10x</c><00:59:17.359><c> or</c><00:59:17.520><c> at</c>\nessentially uh multiplies 10x or at\nessentially uh multiplies 10x or at least<00:59:17.760><c> that's</c><00:59:17.880><c> what</c><00:59:18.000><c> they</c><00:59:18.119><c> try</c><00:59:18.880><c> uh</c><00:59:19.000><c> if</c><00:59:19.119><c> they</c>\nleast that's what they try uh if they\nleast that's what they try uh if they have<00:59:19.599><c> enough</c><00:59:19.839><c> energy</c><00:59:20.319><c> and</c><00:59:20.440><c> if</c><00:59:20.559><c> they</c><00:59:20.640><c> can</c><00:59:20.799><c> buy</c>\nhave enough energy and if they can buy\nhave enough energy and if they can buy enough\nenough\nenough gpus<00:59:23.160><c> uh</c><00:59:23.400><c> great</c><00:59:23.839><c> any</c><00:59:24.039><c> question</c><00:59:24.319><c> on</c><00:59:24.559><c> these</c><00:59:24.960><c> back</c>\ngpus uh great any question on these back\ngpus uh great any question on these back of<00:59:25.280><c> the</c><00:59:25.359><c> envelope</c><00:59:25.760><c> math</c>\nno\nno\nno okay<00:59:32.200><c> so</c><00:59:32.440><c> now</c><00:59:32.599><c> we</c><00:59:32.799><c> talked</c><00:59:33.319><c> about</c><00:59:33.799><c> pre-training</c>\nokay so now we talked about pre-training\nokay so now we talked about pre-training I<00:59:34.880><c> wanted</c><00:59:35.119><c> to</c><00:59:35.280><c> also</c><00:59:35.760><c> chat</c><00:59:36.079><c> about</c><00:59:36.359><c> systems</c>\nI wanted to also chat about systems\nI wanted to also chat about systems because<00:59:37.000><c> now</c><00:59:37.119><c> we</c><00:59:37.280><c> know</c><00:59:37.599><c> computer</c><00:59:38.000><c> is</c><00:59:38.160><c> really</c>\nbecause now we know computer is really\nbecause now we know computer is really important<00:59:38.920><c> so</c><00:59:39.079><c> there's</c><00:59:39.200><c> a</c><00:59:39.359><c> question</c><00:59:39.599><c> of</c><00:59:39.720><c> how</c>\nimportant so there's a question of how\nimportant so there's a question of how do<00:59:39.960><c> you</c><00:59:40.119><c> optimize</c><00:59:40.760><c> the</c><00:59:41.720><c> how</c><00:59:41.799><c> do</c><00:59:41.920><c> you</c><00:59:42.039><c> optimize</c>\ndo you optimize the how do you optimize\ndo you optimize the how do you optimize your<00:59:42.520><c> computer</c><00:59:43.160><c> I</c><00:59:43.240><c> will</c><00:59:43.400><c> leave</c><00:59:43.640><c> that</c><00:59:43.760><c> for</c><00:59:44.000><c> the</c>\nyour computer I will leave that for the\nyour computer I will leave that for the end<00:59:44.559><c> because</c><00:59:44.680><c> I'm</c><00:59:44.799><c> not</c><00:59:44.920><c> sure</c><00:59:45.119><c> how</c><00:59:45.240><c> much</c><00:59:45.400><c> time</c>\nend because I'm not sure how much time\nend because I'm not sure how much time we<00:59:45.680><c> will</c><00:59:45.920><c> have</c><00:59:46.319><c> I</c><00:59:46.400><c> think</c><00:59:46.520><c> it's</c><00:59:46.720><c> important</c><00:59:47.160><c> but</c>\nwe will have I think it's important but\nwe will have I think it's important but hopefully<00:59:47.920><c> I</c><00:59:48.079><c> I'll</c><00:59:48.200><c> be</c><00:59:48.319><c> able</c><00:59:48.559><c> to</c><00:59:49.039><c> to</c><00:59:49.240><c> talk</c>\nhopefully I I'll be able to to talk\nhopefully I I'll be able to to talk about<00:59:49.599><c> it</c><00:59:49.799><c> later</c><00:59:50.440><c> it's</c><00:59:50.680><c> slightly</c><00:59:51.160><c> different</c>\nabout it later it's slightly different\nabout it later it's slightly different than<00:59:52.400><c> what</c><00:59:52.520><c> we've</c><00:59:52.680><c> been</c><00:59:52.839><c> talking</c><00:59:53.119><c> about</c><00:59:53.400><c> right</c>\nthan what we've been talking about right\nthan what we've been talking about right now<00:59:54.039><c> so</c><00:59:54.160><c> I'll</c><00:59:54.319><c> move</c><00:59:54.520><c> on</c><00:59:54.599><c> to</c><00:59:54.799><c> post</c><00:59:55.039><c> training</c><00:59:55.359><c> for</c>\nnow so I'll move on to post training for\nnow so I'll move on to post training for now\nnow\nnow so<00:59:56.799><c> the</c><00:59:56.960><c> task</c><00:59:57.200><c> of</c><00:59:57.319><c> post</c><00:59:57.640><c> training</c><00:59:58.640><c> ER</c><00:59:59.200><c> the</c>\nso the task of post training ER the\nso the task of post training ER the reason<00:59:59.599><c> why</c><00:59:59.720><c> we</c><00:59:59.799><c> need</c><00:59:59.920><c> to</c><01:00:00.039><c> do</c><01:00:00.160><c> Post</c><01:00:00.400><c> training</c>\nreason why we need to do Post training\nreason why we need to do Post training is<01:00:01.240><c> as</c><01:00:01.359><c> I</c><01:00:01.480><c> told</c><01:00:01.640><c> you</c><01:00:01.839><c> before</c><01:00:03.000><c> um</c><01:00:04.000><c> it's</c><01:00:04.160><c> to</c><01:00:04.359><c> make</c>\nis as I told you before um it's to make\nis as I told you before um it's to make AI<01:00:05.319><c> assistants</c><01:00:06.079><c> so</c><01:00:06.319><c> language</c><01:00:06.720><c> modeling</c><01:00:07.559><c> is</c>\nAI assistants so language modeling is\nAI assistants so language modeling is not<01:00:08.839><c> uh</c><01:00:09.079><c> really</c><01:00:09.359><c> the</c><01:00:09.559><c> thing</c><01:00:09.799><c> that</c><01:00:09.920><c> you</c><01:00:10.039><c> want</c>\nnot uh really the thing that you want\nnot uh really the thing that you want when<01:00:10.559><c> you</c><01:00:10.680><c> have</c><01:00:10.799><c> an</c><01:00:10.960><c> AI</c><01:00:11.480><c> assistant</c><01:00:12.480><c> uh</c><01:00:12.599><c> for</c>\nwhen you have an AI assistant uh for\nwhen you have an AI assistant uh for example<01:00:13.160><c> if</c><01:00:13.240><c> you</c><01:00:13.520><c> ask</c><01:00:13.880><c> to</c><01:00:14.079><c> gbd3</c><01:00:14.720><c> which</c><01:00:14.839><c> is</c><01:00:14.920><c> a</c>\nexample if you ask to gbd3 which is a\nexample if you ask to gbd3 which is a purely<01:00:15.799><c> language</c><01:00:16.200><c> Model</c><01:00:16.799><c> A</c><01:00:16.920><c> pure</c><01:00:17.119><c> language</c>\npurely language Model A pure language\npurely language Model A pure language model<01:00:17.760><c> not</c><01:00:18.000><c> a</c><01:00:18.400><c> um</c><01:00:18.880><c> not</c><01:00:19.079><c> an</c><01:00:19.280><c> aligned</c><01:00:19.680><c> one</c><01:00:20.200><c> if</c><01:00:20.280><c> you</c>\nmodel not a um not an aligned one if you\nmodel not a um not an aligned one if you ask<01:00:20.599><c> a</c><01:00:20.760><c> question</c><01:00:21.000><c> like</c><01:00:21.200><c> explain</c><01:00:21.520><c> the</c><01:00:21.640><c> moon</c>\nask a question like explain the moon\nask a question like explain the moon landing<01:00:22.520><c> to</c><01:00:22.640><c> a</c>\nlanding to a\nlanding to a six-year-old<01:00:24.640><c> the</c><01:00:24.880><c> completion</c><01:00:25.359><c> that</c><01:00:25.480><c> you</c>\nsix-year-old the completion that you\nsix-year-old the completion that you would<01:00:25.760><c> get</c><01:00:26.319><c> is</c><01:00:26.520><c> something</c><01:00:26.799><c> like</c><01:00:26.960><c> explain</c><01:00:27.280><c> the</c>\nwould get is something like explain the\nwould get is something like explain the theory<01:00:27.680><c> of</c><01:00:27.880><c> gravity</c><01:00:28.280><c> to</c><01:00:28.400><c> a</c><01:00:28.520><c> six-year-old</c>\ntheory of gravity to a six-year-old\ntheory of gravity to a six-year-old because<01:00:29.559><c> what</c><01:00:29.640><c> it</c><01:00:29.799><c> learned</c><01:00:30.160><c> is</c><01:00:30.240><c> that</c><01:00:30.480><c> on</c><01:00:30.760><c> on</c><01:00:30.960><c> on</c>\nbecause what it learned is that on on on\nbecause what it learned is that on on on internet<01:00:31.559><c> if</c><01:00:31.640><c> you</c><01:00:31.799><c> have</c><01:00:32.160><c> one</c><01:00:32.480><c> question</c><01:00:33.039><c> you</c>\ninternet if you have one question you\ninternet if you have one question you usually<01:00:33.599><c> have</c><01:00:34.079><c> maybe</c><01:00:34.319><c> another</c><01:00:34.599><c> bullet</c><01:00:34.920><c> point</c>\nusually have maybe another bullet point\nusually have maybe another bullet point of<01:00:35.520><c> other</c><01:00:35.799><c> similar</c><01:00:36.200><c> questions</c><01:00:36.839><c> you</c><01:00:36.960><c> don't</c>\nof other similar questions you don't\nof other similar questions you don't usually<01:00:37.440><c> have</c><01:00:37.599><c> question</c><01:00:37.839><c> and</c><01:00:37.960><c> then</c><01:00:38.160><c> answer</c>\nusually have question and then answer\nusually have question and then answer later<01:00:39.319><c> uh</c><01:00:39.480><c> this</c><01:00:39.599><c> is</c><01:00:39.760><c> not</c><01:00:39.960><c> what</c><01:00:40.079><c> you</c><01:00:40.200><c> want</c><01:00:40.799><c> from</c>\nlater uh this is not what you want from\nlater uh this is not what you want from an<01:00:41.359><c> AI</c><01:00:41.960><c> assistant</c><01:00:42.960><c> so</c><01:00:43.240><c> how</c><01:00:43.400><c> do</c><01:00:43.599><c> we</c><01:00:44.480><c> uh</c><01:00:44.599><c> do</c><01:00:44.880><c> this</c>\nan AI assistant so how do we uh do this\nan AI assistant so how do we uh do this alignment<01:00:45.839><c> which</c><01:00:45.920><c> is</c><01:00:46.119><c> this</c><01:00:46.280><c> post</c><01:00:46.559><c> training</c>\nalignment which is this post training\nalignment which is this post training and<01:00:47.079><c> making</c><01:00:47.400><c> these</c><01:00:47.559><c> models</c>\nand making these models\nand making these models assistance<01:00:49.520><c> um</c><01:00:49.839><c> so</c><01:00:50.039><c> the</c><01:00:50.200><c> goal</c><01:00:50.760><c> of</c><01:00:51.240><c> this</c>\nassistance um so the goal of this\nassistance um so the goal of this alignment<01:00:52.000><c> is</c><01:00:52.119><c> to</c><01:00:52.319><c> basically</c><01:00:52.640><c> get</c><01:00:52.799><c> LMS</c><01:00:53.480><c> follow</c>\nalignment is to basically get LMS follow\nalignment is to basically get LMS follow the<01:00:54.400><c> instructions</c><01:00:55.280><c> that</c><01:00:55.400><c> are</c><01:00:55.559><c> given</c><01:00:56.240><c> um</c><01:00:56.599><c> by</c>\nthe instructions that are given um by\nthe instructions that are given um by users<01:00:57.799><c> and</c><01:00:58.200><c> and</c><01:00:58.440><c> maybe</c><01:00:59.039><c> some</c><01:00:59.400><c> designers</c><01:01:00.400><c> kind</c>\nusers and and maybe some designers kind\nusers and and maybe some designers kind of<01:01:00.920><c> desires</c><01:01:01.920><c> um</c><01:01:02.400><c> so</c><01:01:02.839><c> think</c><01:01:03.039><c> about</c><01:01:03.240><c> moderation</c>\nof desires um so think about moderation\nof desires um so think about moderation you<01:01:04.000><c> don't</c><01:01:04.119><c> want</c><01:01:04.240><c> the</c><01:01:04.359><c> model</c><01:01:04.839><c> like</c><01:01:05.280><c> open</c><01:01:05.559><c> ey</c>\nyou don't want the model like open ey\nyou don't want the model like open ey definitely<01:01:05.960><c> doesn't</c><01:01:06.160><c> want</c><01:01:06.319><c> the</c><01:01:06.440><c> model</c><01:01:06.640><c> to</c><01:01:06.760><c> say</c>\ndefinitely doesn't want the model to say\ndefinitely doesn't want the model to say stuff<01:01:07.440><c> that</c><01:01:07.559><c> is</c><01:01:07.720><c> very</c>\nstuff that is very\nstuff that is very toxic<01:01:09.760><c> um</c><01:01:10.119><c> so</c><01:01:10.400><c> here</c><01:01:10.520><c> you</c><01:01:10.599><c> see</c><01:01:10.799><c> on</c><01:01:10.880><c> the</c><01:01:11.000><c> left</c>\ntoxic um so here you see on the left\ntoxic um so here you see on the left hand<01:01:11.440><c> side</c><01:01:12.079><c> uh</c><01:01:12.359><c> that</c><01:01:12.480><c> when</c><01:01:12.599><c> you</c><01:01:12.760><c> ask</c><01:01:12.920><c> a</c>\nhand side uh that when you ask a\nhand side uh that when you ask a question<01:01:13.400><c> it</c><01:01:13.559><c> actually</c><01:01:13.880><c> provides</c><01:01:14.280><c> a</c><01:01:14.680><c> a</c><01:01:14.799><c> real</c>\nquestion it actually provides a a real\nquestion it actually provides a a real answer<01:01:15.400><c> so</c><01:01:15.559><c> it's</c><01:01:15.720><c> not</c><01:01:16.000><c> like</c><01:01:16.400><c> uh</c><01:01:16.559><c> before</c><01:01:16.839><c> the</c>\nanswer so it's not like uh before the\nanswer so it's not like uh before the llm<01:01:17.960><c> and</c><01:01:18.200><c> on</c><01:01:18.319><c> the</c><01:01:18.480><c> right</c><01:01:18.720><c> hand</c><01:01:18.960><c> side</c><01:01:19.440><c> you</c><01:01:19.559><c> see</c>\nllm and on the right hand side you see\nllm and on the right hand side you see that<01:01:20.039><c> it</c><01:01:20.200><c> would</c><01:01:20.680><c> if</c><01:01:20.799><c> you</c><01:01:21.039><c> ask</c><01:01:21.400><c> to</c><01:01:21.599><c> write</c><01:01:21.760><c> a</c>\nthat it would if you ask to write a\nthat it would if you ask to write a tweet<01:01:22.200><c> describing</c><01:01:22.799><c> how</c><01:01:23.440><c> a</c><01:01:23.760><c> certain</c><01:01:24.760><c> part</c><01:01:24.920><c> of</c>\ntweet describing how a certain part of\ntweet describing how a certain part of the<01:01:25.200><c> population</c><01:01:25.720><c> are</c><01:01:26.039><c> evil</c><01:01:26.599><c> it</c><01:01:26.720><c> will</c><01:01:26.920><c> say</c><01:01:27.119><c> that</c>\nthe population are evil it will say that\nthe population are evil it will say that it<01:01:27.319><c> cannot</c><01:01:27.640><c> do</c><01:01:27.839><c> that</c><01:01:29.079><c> um</c><01:01:30.079><c> so</c><01:01:30.400><c> that's</c><01:01:30.720><c> kind</c><01:01:30.839><c> of</c>\nit cannot do that um so that's kind of\nit cannot do that um so that's kind of this\nthis\nthis alignment<01:01:32.720><c> uh</c><01:01:32.839><c> the</c><01:01:32.960><c> background</c><01:01:33.480><c> here</c><01:01:34.039><c> is</c><01:01:34.559><c> that</c>\nalignment uh the background here is that\nalignment uh the background here is that uh<01:01:36.920><c> basically</c><01:01:37.400><c> the</c><01:01:37.640><c> data</c><01:01:37.960><c> that</c><01:01:38.079><c> you</c><01:01:38.200><c> want</c><01:01:38.440><c> for</c>\nuh basically the data that you want for\nuh basically the data that you want for training<01:01:38.920><c> some</c><01:01:39.079><c> of</c><01:01:39.200><c> these</c><01:01:39.440><c> models</c><01:01:40.440><c> um</c><01:01:41.240><c> is</c><01:01:41.799><c> like</c>\ntraining some of these models um is like\ntraining some of these models um is like we<01:01:42.079><c> know</c><01:01:42.280><c> what</c><01:01:42.440><c> we</c><01:01:42.559><c> want</c><01:01:42.880><c> which</c><01:01:43.000><c> is</c><01:01:43.160><c> just</c>\nwe know what we want which is just\nwe know what we want which is just asking<01:01:43.720><c> humans</c><01:01:44.079><c> this</c><01:01:44.160><c> is</c><01:01:44.280><c> a</c><01:01:44.440><c> question</c><01:01:44.680><c> this</c><01:01:44.799><c> is</c>\nasking humans this is a question this is\nasking humans this is a question this is the<01:01:45.039><c> answer</c><01:01:45.280><c> that</c><01:01:45.400><c> you</c><01:01:45.520><c> want</c><01:01:46.400><c> uh</c><01:01:46.559><c> but</c><01:01:46.680><c> the</c>\nthe answer that you want uh but the\nthe answer that you want uh but the thing<01:01:46.920><c> is</c><01:01:47.039><c> that</c><01:01:47.160><c> it's</c><01:01:47.319><c> very</c><01:01:47.480><c> expensive</c><01:01:47.880><c> to</c>\nthing is that it's very expensive to\nthing is that it's very expensive to collect<01:01:48.319><c> that</c><01:01:48.480><c> data</c><01:01:49.039><c> and</c><01:01:49.160><c> it's</c><01:01:49.359><c> hard</c><01:01:49.520><c> to</c><01:01:49.680><c> find</c>\ncollect that data and it's hard to find\ncollect that data and it's hard to find it<01:01:50.319><c> online</c><01:01:51.319><c> uh</c><01:01:51.480><c> in</c><01:01:51.720><c> contrast</c><01:01:52.279><c> pre-training</c>\nit online uh in contrast pre-training\nit online uh in contrast pre-training data<01:01:53.359><c> is</c><01:01:53.520><c> not</c><01:01:53.760><c> what</c><01:01:53.880><c> you</c><01:01:54.000><c> want</c><01:01:54.680><c> but</c><01:01:54.880><c> there's</c><01:01:55.039><c> a</c>\ndata is not what you want but there's a\ndata is not what you want but there's a lot<01:01:55.319><c> of</c><01:01:55.440><c> it</c><01:01:56.160><c> um</c><01:01:56.599><c> so</c><01:01:57.000><c> what</c><01:01:57.279><c> what</c><01:01:57.400><c> we</c><01:01:57.480><c> will</c><01:01:57.640><c> do</c><01:01:57.799><c> a</c>\nlot of it um so what what we will do a\nlot of it um so what what we will do a the<01:01:58.039><c> main</c><01:01:58.279><c> idea</c><01:01:58.839><c> is</c><01:01:59.039><c> simply</c><01:01:59.640><c> take</c><01:01:59.799><c> a</c><01:01:59.960><c> pre-train</c>\nthe main idea is simply take a pre-train\nthe main idea is simply take a pre-train large<01:02:00.680><c> language</c><01:02:01.000><c> model</c><01:02:01.400><c> pre-train</c><01:02:01.920><c> all</c><01:02:02.079><c> of</c>\nlarge language model pre-train all of\nlarge language model pre-train all of internet<01:02:02.680><c> and</c><01:02:02.760><c> then</c><01:02:02.880><c> you</c><01:02:03.000><c> just</c><01:02:03.119><c> fine</c><01:02:03.359><c> tune</c><01:02:03.640><c> so</c>\ninternet and then you just fine tune so\ninternet and then you just fine tune so you<01:02:03.880><c> just</c><01:02:04.000><c> change</c><01:02:04.240><c> a</c><01:02:04.359><c> little</c><01:02:04.520><c> bit</c><01:02:04.640><c> of</c><01:02:04.760><c> weights</c>\nyou just change a little bit of weights\nyou just change a little bit of weights on<01:02:05.400><c> the</c><01:02:05.559><c> type</c><01:02:05.720><c> of</c><01:02:05.880><c> data</c><01:02:06.119><c> that</c><01:02:06.279><c> you</c><01:02:06.480><c> actually</c>\non the type of data that you actually\non the type of data that you actually want<01:02:07.440><c> and</c><01:02:07.640><c> hopefully</c><01:02:08.119><c> given</c><01:02:08.359><c> it</c><01:02:08.520><c> you</c><01:02:08.640><c> already</c>\nwant and hopefully given it you already\nwant and hopefully given it you already pre-train<01:02:09.240><c> it</c><01:02:09.359><c> on</c><01:02:09.440><c> all</c><01:02:09.640><c> of</c><01:02:09.760><c> Internet</c><01:02:10.279><c> it</c>\npre-train it on all of Internet it\npre-train it on all of Internet it basically<01:02:10.839><c> learns</c><01:02:11.680><c> or</c><01:02:11.880><c> knows</c><01:02:12.079><c> how</c><01:02:12.200><c> to</c><01:02:12.319><c> speak</c>\nbasically learns or knows how to speak\nbasically learns or knows how to speak in<01:02:12.839><c> English</c><01:02:13.240><c> and</c><01:02:13.480><c> and</c><01:02:14.000><c> knows</c><01:02:14.799><c> a</c><01:02:15.039><c> standard</c><01:02:16.039><c> um</c>\nin English and and knows a standard um\nin English and and knows a standard um language<01:02:17.359><c> syntax</c><01:02:18.359><c> uh</c><01:02:18.520><c> then</c><01:02:18.640><c> you</c><01:02:18.760><c> can</c><01:02:19.079><c> really</c>\nlanguage syntax uh then you can really\nlanguage syntax uh then you can really find<01:02:20.000><c> tune</c><01:02:20.200><c> in</c><01:02:20.359><c> with</c><01:02:20.520><c> very</c><01:02:20.720><c> little</c>\nfind tune in with very little\nfind tune in with very little data<01:02:23.440><c> okay</c><01:02:23.720><c> sft</c><01:02:24.720><c> so</c><01:02:24.920><c> supervis</c><01:02:25.400><c> fine</c><01:02:25.559><c> tuning</c><01:02:26.240><c> is</c>\ndata okay sft so supervis fine tuning is\ndata okay sft so supervis fine tuning is really<01:02:26.680><c> exactly</c><01:02:27.039><c> what</c><01:02:27.119><c> I</c><01:02:27.279><c> just</c><01:02:27.400><c> said</c><01:02:27.680><c> which</c><01:02:27.760><c> is</c>\nreally exactly what I just said which is\nreally exactly what I just said which is the<01:02:28.079><c> idea</c><01:02:28.400><c> of</c><01:02:28.559><c> fine-tuning</c><01:02:29.000><c> the</c><01:02:29.119><c> large</c>\nthe idea of fine-tuning the large\nthe idea of fine-tuning the large language<01:02:29.680><c> model</c><01:02:30.440><c> on</c><01:02:31.319><c> uh</c><01:02:31.520><c> basically</c><01:02:31.960><c> the</c>\nlanguage model on uh basically the\nlanguage model on uh basically the desired<01:02:32.680><c> answers</c><01:02:33.079><c> that</c><01:02:33.200><c> are</c><01:02:33.319><c> collected</c><01:02:33.680><c> from</c>\ndesired answers that are collected from\ndesired answers that are collected from humans<01:02:35.200><c> um</c><01:02:35.760><c> so</c><01:02:36.039><c> why</c><01:02:36.200><c> is</c><01:02:36.279><c> it</c><01:02:36.440><c> called</c><01:02:36.640><c> supervis</c>\nhumans um so why is it called supervis\nhumans um so why is it called supervis fine<01:02:37.240><c> tuning</c><01:02:37.760><c> because</c><01:02:38.160><c> you</c><01:02:38.359><c> basically</c><01:02:38.640><c> want</c>\nfine tuning because you basically want\nfine tuning because you basically want to<01:02:38.839><c> do</c><01:02:39.520><c> language</c><01:02:39.920><c> modeling</c><01:02:40.599><c> on</c><01:02:40.799><c> the</c><01:02:40.960><c> real</c>\nto do language modeling on the real\nto do language modeling on the real ansers<01:02:41.559><c> so</c><01:02:41.680><c> language</c><01:02:42.039><c> modeling</c><01:02:42.359><c> is</c><01:02:42.480><c> this</c><01:02:42.640><c> like</c>\nansers so language modeling is this like\nansers so language modeling is this like next<01:02:43.039><c> word</c><01:02:43.279><c> prediction</c><01:02:44.279><c> and</c><01:02:44.599><c> and</c><01:02:44.720><c> that's</c><01:02:44.839><c> the</c>\nnext word prediction and and that's the\nnext word prediction and and that's the fine-tuning<01:02:45.480><c> part</c><01:02:45.839><c> and</c><01:02:45.960><c> then</c><01:02:46.119><c> you</c><01:02:46.200><c> want</c><01:02:46.319><c> to</c><01:02:46.480><c> do</c>\nfine-tuning part and then you want to do\nfine-tuning part and then you want to do it<01:02:46.760><c> on</c><01:02:47.240><c> desired</c><01:02:47.680><c> answers</c><01:02:48.039><c> given</c><01:02:48.279><c> by</c><01:02:48.400><c> humans</c><01:02:48.680><c> so</c>\nit on desired answers given by humans so\nit on desired answers given by humans so that's<01:02:48.960><c> why</c><01:02:49.079><c> we</c><01:02:49.160><c> call</c><01:02:49.279><c> it</c>\nthat's why we call it\nthat's why we call it supervis<01:02:51.400><c> so</c><01:02:51.559><c> how</c><01:02:51.680><c> do</c><01:02:51.799><c> we</c><01:02:51.880><c> collect</c><01:02:52.240><c> this</c><01:02:52.400><c> data</c>\nsupervis so how do we collect this data\nsupervis so how do we collect this data well<01:02:53.119><c> we</c><01:02:53.319><c> I</c><01:02:53.440><c> just</c><01:02:53.559><c> said</c><01:02:53.760><c> it</c><01:02:54.000><c> you</c><01:02:54.279><c> just</c><01:02:54.520><c> ask</c>\nwell we I just said it you just ask\nwell we I just said it you just ask humans<01:02:55.520><c> uh</c><01:02:55.640><c> to</c><01:02:55.839><c> to</c><01:02:55.920><c> tell</c><01:02:56.079><c> you</c><01:02:56.319><c> this</c><01:02:56.400><c> is</c><01:02:56.599><c> the</c>\nhumans uh to to tell you this is the\nhumans uh to to tell you this is the this<01:02:57.039><c> is</c><01:02:57.119><c> a</c><01:02:57.319><c> question</c><01:02:57.640><c> this</c><01:02:57.760><c> is</c><01:02:57.839><c> the</c><01:02:58.000><c> answer</c>\nthis is a question this is the answer\nthis is a question this is the answer that<01:02:58.440><c> you</c><01:02:59.119><c> uh</c><01:02:59.240><c> you</c><01:02:59.359><c> would</c><01:02:59.520><c> want</c><01:02:59.680><c> from</c><01:02:59.839><c> some</c><01:03:00.000><c> of</c>\nthat you uh you would want from some of\nthat you uh you would want from some of these<01:03:00.319><c> models</c><01:03:00.960><c> so</c><01:03:01.240><c> this</c><01:03:01.319><c> is</c><01:03:01.440><c> an</c><01:03:01.720><c> example</c><01:03:02.720><c> um</c>\nthese models so this is an example um\nthese models so this is an example um sorry<01:03:03.279><c> I</c><01:03:03.400><c> can't</c><01:03:03.599><c> read</c><01:03:03.920><c> very</c><01:03:04.039><c> well</c><01:03:04.240><c> on</c><01:03:04.359><c> my</c>\nsorry I can't read very well on my\nsorry I can't read very well on my computer<01:03:05.000><c> but</c><01:03:05.680><c> uh</c><01:03:05.839><c> my</c><01:03:06.119><c> kid</c><01:03:06.920><c> uh</c><01:03:07.039><c> needs</c><01:03:07.279><c> to</c><01:03:07.440><c> do</c><01:03:07.599><c> a</c>\ncomputer but uh my kid uh needs to do a\ncomputer but uh my kid uh needs to do a science<01:03:08.480><c> um</c><01:03:08.680><c> no</c><01:03:08.799><c> let's</c><01:03:08.960><c> read</c><01:03:09.200><c> this</c><01:03:09.319><c> one</c><01:03:09.720><c> can</c>\nscience um no let's read this one can\nscience um no let's read this one can you<01:03:10.079><c> write</c><01:03:10.440><c> a</c><01:03:10.640><c> short</c><01:03:11.079><c> introduction</c><01:03:11.799><c> about</c><01:03:11.960><c> the</c>\nyou write a short introduction about the\nyou write a short introduction about the relevance<01:03:12.480><c> of</c><01:03:12.599><c> the</c><01:03:12.720><c> term</c><01:03:12.960><c> monopsony</c><01:03:13.920><c> and</c><01:03:14.039><c> then</c>\nrelevance of the term monopsony and then\nrelevance of the term monopsony and then it<01:03:14.279><c> says</c><01:03:14.440><c> monopsony</c><01:03:14.960><c> refers</c><01:03:15.279><c> to</c><01:03:15.400><c> a</c><01:03:15.480><c> market</c>\nit says monopsony refers to a market\nit says monopsony refers to a market structure<01:03:16.119><c> blah</c><01:03:16.279><c> blah</c><01:03:16.480><c> blah</c><01:03:16.640><c> and</c><01:03:16.720><c> that's</c><01:03:16.799><c> a</c>\nstructure blah blah blah and that's a\nstructure blah blah blah and that's a human<01:03:17.160><c> that</c><01:03:17.319><c> wrote</c><01:03:17.920><c> that</c><01:03:18.920><c> um</c><01:03:19.359><c> so</c><01:03:19.559><c> actually</c>\nhuman that wrote that um so actually\nhuman that wrote that um so actually this<01:03:19.839><c> is</c><01:03:20.000><c> open</c><01:03:20.279><c> Assistant</c><01:03:20.839><c> which</c><01:03:21.000><c> was</c><01:03:21.200><c> a</c><01:03:21.680><c> a</c><01:03:21.960><c> way</c>\nthis is open Assistant which was a a way\nthis is open Assistant which was a a way to<01:03:22.559><c> collect</c><01:03:24.000><c> um</c><01:03:25.000><c> uh</c><01:03:25.279><c> data</c><01:03:25.799><c> online</c><01:03:26.520><c> by</c>\nto collect um uh data online by\nto collect um uh data online by humans<01:03:28.359><c> so</c><01:03:28.839><c> this</c><01:03:29.279><c> type</c><01:03:29.520><c> of</c><01:03:29.839><c> supervised</c><01:03:30.359><c> fine</c>\nhumans so this type of supervised fine\nhumans so this type of supervised fine tuning<01:03:30.760><c> or</c><01:03:30.920><c> alignment</c><01:03:31.520><c> is</c><01:03:31.720><c> really</c><01:03:32.000><c> the</c><01:03:32.160><c> key</c><01:03:32.520><c> of</c>\ntuning or alignment is really the key of\ntuning or alignment is really the key of Chad<01:03:33.000><c> GPT</c><01:03:34.000><c> this</c><01:03:34.160><c> is</c><01:03:34.400><c> what</c><01:03:34.599><c> made</c><01:03:35.319><c> uh</c><01:03:35.440><c> the</c><01:03:35.599><c> big</c>\nChad GPT this is what made uh the big\nChad GPT this is what made uh the big jump<01:03:36.160><c> from</c><01:03:36.400><c> gpt3</c><01:03:37.200><c> which</c><01:03:37.319><c> was</c><01:03:37.480><c> mostly</c>\njump from gpt3 which was mostly\njump from gpt3 which was mostly something<01:03:38.119><c> that</c><01:03:38.279><c> was</c><01:03:38.440><c> known</c><01:03:38.799><c> by</c><01:03:38.960><c> AI</c>\nsomething that was known by AI\nsomething that was known by AI researchers<01:03:40.240><c> to</c><01:03:40.480><c> Chad</c><01:03:40.760><c> GPT</c><01:03:41.559><c> which</c><01:03:41.799><c> became</c>\nresearchers to Chad GPT which became\nresearchers to Chad GPT which became known<01:03:42.440><c> by</c><01:03:42.960><c> basically</c>\nknown by basically\nknown by basically everyone\neveryone\neveryone um<01:03:47.319><c> so</c><01:03:48.200><c> the</c><01:03:48.400><c> problem</c><01:03:48.760><c> with</c><01:03:49.720><c> uh</c><01:03:50.720><c> human</c><01:03:51.240><c> data</c><01:03:51.680><c> is</c>\num so the problem with uh human data is\num so the problem with uh human data is that<01:03:52.000><c> it's</c><01:03:52.920><c> uh</c><01:03:53.079><c> very</c><01:03:53.240><c> slow</c><01:03:53.520><c> to</c><01:03:53.680><c> collect</c><01:03:54.160><c> and</c>\nthat it's uh very slow to collect and\nthat it's uh very slow to collect and very<01:03:54.559><c> expensive</c><01:03:56.000><c> um</c><01:03:56.440><c> so</c>\nvery expensive um so\nvery expensive um so one<01:03:58.240><c> possible</c><01:03:58.839><c> simple</c><01:03:59.359><c> idea</c><01:03:59.920><c> is</c><01:04:00.079><c> to</c><01:04:00.240><c> use</c><01:04:00.520><c> llms</c>\none possible simple idea is to use llms\none possible simple idea is to use llms to<01:04:01.599><c> scale</c><01:04:02.039><c> data</c><01:04:02.319><c> collection</c><01:04:03.279><c> uh</c><01:04:03.359><c> so</c><01:04:03.559><c> that's</c>\nto scale data collection uh so that's\nto scale data collection uh so that's exactly<01:04:04.119><c> what</c><01:04:04.240><c> we</c><01:04:04.400><c> did</c><01:04:04.559><c> with</c><01:04:04.760><c> alpaca</c><01:04:05.760><c> uh</c><01:04:05.920><c> one</c>\nexactly what we did with alpaca uh one\nexactly what we did with alpaca uh one year<01:04:06.279><c> ago</c><01:04:06.839><c> what</c><01:04:06.960><c> we</c><01:04:07.119><c> did</c><01:04:07.279><c> is</c><01:04:07.400><c> that</c><01:04:07.599><c> we</c><01:04:07.799><c> asked</c><01:04:08.440><c> uh</c>\nyear ago what we did is that we asked uh\nyear ago what we did is that we asked uh humans<01:04:08.920><c> or</c><01:04:09.119><c> we</c><01:04:09.240><c> use</c><01:04:09.440><c> a</c><01:04:09.599><c> data</c><01:04:09.839><c> set</c><01:04:10.000><c> of</c><01:04:10.160><c> human</c><01:04:10.920><c> uh</c>\nhumans or we use a data set of human uh\nhumans or we use a data set of human uh question<01:04:11.400><c> answers</c><01:04:11.960><c> so</c><01:04:12.160><c> there</c><01:04:12.240><c> were</c><01:04:12.920><c> 175</c><01:04:13.920><c> uh</c>\nquestion answers so there were 175 uh\nquestion answers so there were 175 uh question<01:04:14.359><c> answers</c><01:04:14.799><c> here</c><01:04:15.240><c> and</c><01:04:15.359><c> we</c><01:04:15.520><c> asked</c><01:04:15.760><c> the</c>\nquestion answers here and we asked the\nquestion answers here and we asked the best<01:04:16.039><c> mod</c><01:04:16.279><c> at</c><01:04:16.400><c> the</c><01:04:16.520><c> time</c><01:04:16.680><c> so</c><01:04:17.319><c> text3</c><01:04:18.319><c> to</c>\nbest mod at the time so text3 to\nbest mod at the time so text3 to basically<01:04:18.920><c> generate</c><01:04:19.760><c> many</c><01:04:20.039><c> more</c><01:04:20.680><c> of</c><01:04:20.920><c> these</c>\nbasically generate many more of these\nbasically generate many more of these question<01:04:21.400><c> and</c><01:04:21.640><c> answers</c><01:04:22.319><c> so</c><01:04:22.480><c> all</c><01:04:22.599><c> we</c><01:04:22.760><c> did</c><01:04:22.920><c> is</c>\nquestion and answers so all we did is\nquestion and answers so all we did is like<01:04:23.359><c> this</c><01:04:23.480><c> is</c><01:04:23.799><c> what</c><01:04:23.960><c> humans</c><01:04:24.240><c> would</c><01:04:24.480><c> write</c><01:04:24.880><c> now</c>\nlike this is what humans would write now\nlike this is what humans would write now write<01:04:25.359><c> similar</c><01:04:26.039><c> answers</c><01:04:26.359><c> and</c><01:04:26.520><c> similar</c>\nwrite similar answers and similar\nwrite similar answers and similar questions<01:04:27.640><c> and</c><01:04:27.760><c> we</c><01:04:28.000><c> collected</c><01:04:29.200><c> 52,000</c><01:04:30.200><c> LM</c>\nquestions and we collected 52,000 LM\nquestions and we collected 52,000 LM generated<01:04:31.440><c> question</c><01:04:31.799><c> answers</c><01:04:32.520><c> and</c><01:04:32.640><c> then</c><01:04:32.799><c> what</c>\ngenerated question answers and then what\ngenerated question answers and then what we<01:04:33.079><c> did</c><01:04:33.200><c> is</c><01:04:33.359><c> simply</c><01:04:33.760><c> we</c><01:04:33.880><c> took</c><01:04:34.039><c> Lama</c><01:04:34.400><c> 7B</c><01:04:34.799><c> which</c>\nwe did is simply we took Lama 7B which\nwe did is simply we took Lama 7B which was<01:04:35.039><c> the</c><01:04:35.240><c> best</c><01:04:35.440><c> pre-train</c><01:04:35.880><c> model</c><01:04:36.119><c> at</c><01:04:36.200><c> the</c><01:04:36.359><c> time</c>\nwas the best pre-train model at the time\nwas the best pre-train model at the time and<01:04:36.839><c> we</c><01:04:36.960><c> just</c><01:04:37.119><c> fine-</c><01:04:37.359><c> tuned</c><01:04:37.920><c> this</c><01:04:38.079><c> with</c>\nand we just fine- tuned this with\nand we just fine- tuned this with supervised<01:04:38.680><c> fine</c><01:04:38.839><c> tuning</c><01:04:39.079><c> as</c><01:04:39.200><c> I</c><01:04:39.319><c> told</c><01:04:39.520><c> you</c><01:04:39.960><c> and</c>\nsupervised fine tuning as I told you and\nsupervised fine tuning as I told you and that's<01:04:40.400><c> how</c><01:04:40.559><c> we</c><01:04:40.720><c> got</c><01:04:41.400><c> um</c><01:04:41.599><c> the</c><01:04:41.720><c> Alpac</c><01:04:42.039><c> s7b</c>\nthat's how we got um the Alpac s7b\nthat's how we got um the Alpac s7b model<01:04:44.559><c> uh</c><01:04:45.119><c> and</c><01:04:45.319><c> this</c><01:04:45.400><c> is</c><01:04:45.520><c> the</c><01:04:45.680><c> type</c><01:04:45.880><c> of</c><01:04:46.000><c> data</c>\nmodel uh and this is the type of data\nmodel uh and this is the type of data that<01:04:46.359><c> we</c><01:04:46.480><c> collected</c><01:04:47.119><c> so</c><01:04:47.319><c> things</c><01:04:47.640><c> like</c><01:04:48.039><c> what</c>\nthat we collected so things like what\nthat we collected so things like what does<01:04:48.559><c> algorithm</c><01:04:49.079><c> mean</c><01:04:49.359><c> an</c><01:04:49.520><c> algorithm</c><01:04:49.960><c> is</c><01:04:50.039><c> a</c>\ndoes algorithm mean an algorithm is a\ndoes algorithm mean an algorithm is a step<01:04:50.440><c> by</c><01:04:50.640><c> a</c><01:04:51.039><c> stepbystep</c><01:04:52.039><c> uh</c><01:04:52.559><c> set</c><01:04:52.799><c> of</c>\nstep by a stepbystep uh set of\nstep by a stepbystep uh set of instruction<01:04:53.400><c> used</c><01:04:53.680><c> to</c><01:04:53.799><c> solve</c><01:04:54.000><c> a</c><01:04:54.160><c> problem</c><01:04:54.559><c> or</c>\ninstruction used to solve a problem or\ninstruction used to solve a problem or achieve<01:04:55.240><c> a</c><01:04:55.359><c> goal</c><01:04:55.680><c> blah</c><01:04:55.839><c> blah</c><01:04:56.000><c> blah</c><01:04:56.160><c> blah</c><01:04:56.440><c> so</c>\nachieve a goal blah blah blah blah so\nachieve a goal blah blah blah blah so the<01:04:56.720><c> data</c><01:04:57.000><c> is</c><01:04:57.160><c> not</c><01:04:57.480><c> actually</c><01:04:57.920><c> it's</c><01:04:58.079><c> actually</c>\nthe data is not actually it's actually\nthe data is not actually it's actually pretty<01:04:58.559><c> good</c><01:04:58.880><c> given</c><01:04:59.160><c> it</c><01:04:59.319><c> was</c><01:04:59.480><c> LM</c><01:04:59.839><c> generated</c><01:05:00.319><c> by</c>\npretty good given it was LM generated by\npretty good given it was LM generated by LMS<01:05:01.000><c> from</c><01:05:01.480><c> essentially</c><01:05:01.880><c> two</c><01:05:02.079><c> generations</c><01:05:03.000><c> ago</c>\nLMS from essentially two generations ago\nLMS from essentially two generations ago um<01:05:05.160><c> so</c><01:05:05.640><c> that</c><01:05:05.839><c> really</c><01:05:06.039><c> started</c><01:05:06.520><c> at</c><01:05:06.640><c> least</c><01:05:06.799><c> for</c>\num so that really started at least for\num so that really started at least for us<01:05:07.240><c> kind</c><01:05:07.359><c> of</c><01:05:07.480><c> as</c><01:05:07.640><c> an</c><01:05:07.960><c> academic</c><01:05:08.440><c> replication</c><01:05:08.880><c> of</c>\nus kind of as an academic replication of\nus kind of as an academic replication of chat<01:05:09.279><c> GPT</c><01:05:10.279><c> uh</c><01:05:10.520><c> now</c><01:05:11.000><c> it</c><01:05:11.440><c> really</c><01:05:12.119><c> there's</c><01:05:12.279><c> a</c><01:05:12.400><c> big</c>\nchat GPT uh now it really there's a big\nchat GPT uh now it really there's a big field<01:05:12.880><c> of</c><01:05:13.039><c> like</c><01:05:13.200><c> synthetic</c><01:05:13.640><c> data</c><01:05:13.920><c> generation</c>\nfield of like synthetic data generation\nfield of like synthetic data generation of<01:05:15.200><c> how</c><01:05:15.359><c> to</c><01:05:15.720><c> use</c><01:05:16.079><c> llms</c><01:05:16.839><c> to</c><01:05:17.039><c> basically</c><01:05:17.520><c> make</c>\nof how to use llms to basically make\nof how to use llms to basically make development<01:05:18.640><c> of</c><01:05:18.839><c> llms</c><01:05:19.720><c> faster</c><01:05:20.720><c> um</c><01:05:21.440><c> and</c><01:05:21.760><c> by</c>\ndevelopment of llms faster um and by\ndevelopment of llms faster um and by basically<01:05:22.279><c> by</c><01:05:22.400><c> decreasing</c><01:05:22.799><c> the</c><01:05:23.000><c> amount</c><01:05:23.279><c> of</c><01:05:23.520><c> of</c>\nbasically by decreasing the amount of of\nbasically by decreasing the amount of of human<01:05:23.880><c> hours</c><01:05:24.160><c> that</c><01:05:24.279><c> you</c><01:05:24.400><c> need</c>\nhuman hours that you need\nhuman hours that you need quantity<01:05:27.440><c> of</c><01:05:27.680><c> data</c><01:05:28.680><c> so</c><01:05:28.839><c> we</c><01:05:29.000><c> talked</c><01:05:29.240><c> about</c><01:05:29.480><c> what</c>\nquantity of data so we talked about what\nquantity of data so we talked about what type<01:05:29.760><c> of</c><01:05:29.880><c> data</c><01:05:30.119><c> and</c><01:05:30.240><c> how</c><01:05:30.359><c> we</c><01:05:30.440><c> collect</c><01:05:30.760><c> it</c><01:05:31.440><c> um</c>\ntype of data and how we collect it um\ntype of data and how we collect it um one<01:05:31.920><c> thing</c><01:05:32.079><c> which</c><01:05:32.200><c> is</c><01:05:32.359><c> surprising</c><01:05:32.839><c> with</c><01:05:33.039><c> sft</c>\none thing which is surprising with sft\none thing which is surprising with sft is<01:05:34.119><c> that</c><01:05:34.240><c> you</c><01:05:34.359><c> don't</c><01:05:34.559><c> need</c><01:05:34.760><c> that</c><01:05:34.920><c> much</c><01:05:35.240><c> data</c><01:05:36.240><c> uh</c>\nis that you don't need that much data uh\nis that you don't need that much data uh so<01:05:37.000><c> what</c><01:05:37.160><c> this</c><01:05:37.319><c> paper</c><01:05:37.559><c> showed</c><01:05:37.880><c> this</c><01:05:38.000><c> is</c><01:05:38.119><c> called</c>\nso what this paper showed this is called\nso what this paper showed this is called Lima<01:05:39.079><c> is</c><01:05:39.279><c> that</c><01:05:39.599><c> if</c><01:05:39.760><c> you</c><01:05:40.039><c> have</c><01:05:40.359><c> if</c><01:05:40.480><c> you</c><01:05:40.640><c> scale</c>\nLima is that if you have if you scale\nLima is that if you have if you scale the<01:05:41.720><c> amount</c><01:05:41.920><c> of</c><01:05:42.079><c> data</c><01:05:42.279><c> that</c><01:05:42.440><c> use</c><01:05:42.799><c> from</c><01:05:43.160><c> uh</c>\nthe amount of data that use from uh\nthe amount of data that use from uh supervised<01:05:43.720><c> fine</c><01:05:43.880><c> training</c><01:05:44.440><c> from</c><01:05:44.640><c> 2,000</c><01:05:45.240><c> to</c>\nsupervised fine training from 2,000 to\nsupervised fine training from 2,000 to 32,000<01:05:46.640><c> it</c><01:05:46.760><c> really</c><01:05:46.960><c> doesn't</c><01:05:47.240><c> help</c><01:05:47.480><c> much</c><01:05:47.880><c> so</c>\n32,000 it really doesn't help much so\n32,000 it really doesn't help much so here<01:05:48.240><c> scaling</c><01:05:48.559><c> laws</c><01:05:48.839><c> definitely</c><01:05:49.119><c> don't</c><01:05:49.359><c> help</c>\nhere scaling laws definitely don't help\nhere scaling laws definitely don't help um<01:05:50.920><c> so</c><01:05:51.279><c> the</c><01:05:51.559><c> the</c><01:05:51.680><c> intuition</c><01:05:52.279><c> here</c><01:05:52.760><c> is</c><01:05:52.920><c> that</c><01:05:53.200><c> all</c>\num so the the intuition here is that all\num so the the intuition here is that all you<01:05:53.720><c> learn</c><01:05:54.720><c> um</c><01:05:55.440><c> is</c><01:05:55.680><c> is</c><01:05:55.920><c> you</c><01:05:56.079><c> learn</c><01:05:56.359><c> how</c><01:05:56.480><c> to</c>\nyou learn um is is you learn how to\nyou learn um is is you learn how to format<01:05:57.559><c> your</c><01:05:57.760><c> desired</c><01:05:58.200><c> answers</c><01:05:58.960><c> another</c><01:05:59.240><c> way</c>\nformat your desired answers another way\nformat your desired answers another way of<01:05:59.480><c> saying</c><01:05:59.799><c> it</c><01:06:00.279><c> is</c><01:06:00.480><c> that</c><01:06:00.599><c> your</c><01:06:00.760><c> pre-trained</c>\nof saying it is that your pre-trained\nof saying it is that your pre-trained models<01:06:02.160><c> they</c><01:06:02.520><c> essentially</c><01:06:03.000><c> model</c><01:06:03.359><c> the</c>\nmodels they essentially model the\nmodels they essentially model the distribution<01:06:04.039><c> of</c><01:06:04.319><c> every</c><01:06:04.559><c> user</c><01:06:04.880><c> on</c><01:06:05.079><c> internet</c>\ndistribution of every user on internet\ndistribution of every user on internet one<01:06:06.039><c> that</c><01:06:06.279><c> might</c><01:06:06.440><c> write</c><01:06:06.680><c> bullet</c><01:06:07.000><c> points</c>\none that might write bullet points\none that might write bullet points another<01:06:07.839><c> one</c><01:06:08.079><c> that</c><01:06:08.240><c> might</c><01:06:08.520><c> answer</c><01:06:08.880><c> qu</c><01:06:09.240><c> answer</c>\nanother one that might answer qu answer\nanother one that might answer qu answer question<01:06:10.000><c> with</c><01:06:10.160><c> an</c><01:06:10.319><c> answer</c><01:06:11.000><c> so</c><01:06:11.200><c> all</c><01:06:11.400><c> you</c><01:06:11.599><c> tell</c>\nquestion with an answer so all you tell\nquestion with an answer so all you tell your<01:06:11.880><c> model</c><01:06:12.240><c> is</c><01:06:12.440><c> like</c><01:06:13.079><c> wait</c><01:06:13.359><c> you</c><01:06:13.480><c> should</c>\nyour model is like wait you should\nyour model is like wait you should actually<01:06:14.039><c> be</c><01:06:14.319><c> optimizing</c><01:06:14.920><c> more</c><01:06:15.119><c> for</c><01:06:15.440><c> this</c>\nactually be optimizing more for this\nactually be optimizing more for this type<01:06:15.799><c> of</c><01:06:15.920><c> user</c><01:06:16.440><c> than</c><01:06:16.640><c> another</c><01:06:16.920><c> one</c><01:06:17.200><c> so</c><01:06:17.359><c> you're</c>\ntype of user than another one so you're\ntype of user than another one so you're not<01:06:17.680><c> actually</c><01:06:17.960><c> teaching</c><01:06:18.480><c> it</c><01:06:18.839><c> and</c><01:06:19.000><c> you're</c><01:06:19.160><c> not</c>\nnot actually teaching it and you're not\nnot actually teaching it and you're not teaching<01:06:19.880><c> anything</c><01:06:20.680><c> through</c><01:06:21.200><c> this</c><01:06:21.680><c> um</c><01:06:22.160><c> sft</c><01:06:23.160><c> uh</c>\nteaching anything through this um sft uh\nteaching anything through this um sft uh so<01:06:23.440><c> supervis</c><01:06:23.880><c> fine</c><01:06:24.079><c> tuning</c><01:06:24.559><c> all</c><01:06:24.680><c> you</c><01:06:24.839><c> do</c><01:06:25.039><c> is</c>\nso supervis fine tuning all you do is\nso supervis fine tuning all you do is you<01:06:25.599><c> tell</c><01:06:25.799><c> the</c><01:06:25.920><c> model</c><01:06:26.200><c> to</c><01:06:26.359><c> kind</c><01:06:26.520><c> of</c><01:06:26.839><c> optimize</c>\nyou tell the model to kind of optimize\nyou tell the model to kind of optimize for<01:06:27.480><c> one</c><01:06:27.640><c> type</c><01:06:27.799><c> of</c><01:06:27.920><c> user</c><01:06:28.279><c> that</c><01:06:28.359><c> it</c><01:06:28.520><c> saw</c><01:06:28.839><c> already</c>\nfor one type of user that it saw already\nfor one type of user that it saw already in<01:06:29.400><c> a</c><01:06:29.559><c> pre-train</c><01:06:30.000><c> data</c><01:06:30.279><c> set</c><01:06:31.119><c> so</c><01:06:31.279><c> the</c><01:06:31.400><c> knowledge</c>\nin a pre-train data set so the knowledge\nin a pre-train data set so the knowledge is<01:06:31.880><c> already</c><01:06:32.079><c> in</c><01:06:32.160><c> the</c><01:06:32.240><c> pre-train</c><01:06:32.640><c> llm</c><01:06:33.520><c> uh</c><01:06:33.720><c> and</c>\nis already in the pre-train llm uh and\nis already in the pre-train llm uh and you<01:06:34.039><c> basically</c><01:06:34.359><c> just</c><01:06:34.480><c> specialize</c><01:06:34.920><c> to</c><01:06:35.039><c> one</c>\nyou basically just specialize to one\nyou basically just specialize to one type<01:06:35.359><c> of</c>\ntype of\ntype of user<01:06:37.760><c> great</c><01:06:38.000><c> any</c><01:06:38.160><c> question</c><01:06:38.400><c> on</c>\nuser great any question on\nuser great any question on sft<01:06:41.200><c> yes</c><01:06:42.279><c> so</c><01:06:43.279><c> I</c><01:06:43.400><c> know</c><01:06:43.520><c> it's</c><01:06:43.640><c> a</c><01:06:43.760><c> big</c><01:06:43.920><c> issue</c><01:06:44.200><c> with</c>\nsft yes so I know it's a big issue with\nsft yes so I know it's a big issue with synthetic<01:06:44.920><c> data</c><01:06:45.319><c> where</c><01:06:46.279><c> uh</c><01:06:46.839><c> if</c><01:06:46.920><c> you</c><01:06:47.119><c> keep</c>\nsynthetic data where uh if you keep\nsynthetic data where uh if you keep generating<01:06:48.279><c> data</c><01:06:48.520><c> from</c><01:06:48.680><c> the</c><01:06:48.799><c> same</c>\ngenerating data from the same\ngenerating data from the same distribution<01:06:49.599><c> eventually</c><01:06:49.960><c> you're</c><01:06:50.119><c> not</c>\ndistribution eventually you're not\ndistribution eventually you're not learning<01:06:50.640><c> a</c><01:06:50.760><c> new</c><01:06:50.960><c> distribution</c><01:06:51.480><c> you're</c>\nlearning a new distribution you're\nlearning a new distribution you're essentially<01:06:52.079><c> playing</c><01:06:52.359><c> with</c><01:06:52.480><c> it</c><01:06:52.599><c> it</c><01:06:52.760><c> just</c>\nessentially playing with it it just\nessentially playing with it it just bootstrapping<01:06:53.559><c> that</c><01:06:54.039><c> yeah</c><01:06:55.039><c> surely</c>\nbootstrapping that yeah surely\nbootstrapping that yeah surely you<01:06:56.240><c> can't</c><01:06:56.480><c> scale</c><01:06:56.839><c> that</c><01:06:56.960><c> forever</c><01:06:57.559><c> right</c><01:06:57.680><c> you</c>\nyou can't scale that forever right you\nyou can't scale that forever right you can't<01:06:58.079><c> keep</c><01:06:58.359><c> going</c><01:06:58.559><c> on</c><01:06:58.799><c> and</c><01:06:58.960><c> generating</c><01:06:59.440><c> from</c>\ncan't keep going on and generating from\ncan't keep going on and generating from the<01:06:59.680><c> same</c><01:06:59.880><c> distribution</c><01:07:00.400><c> you</c><01:07:00.520><c> hope</c><01:07:00.640><c> to</c><01:07:00.760><c> learn</c>\nthe same distribution you hope to learn\nthe same distribution you hope to learn something<01:07:01.400><c> new</c><01:07:01.760><c> yeah</c><01:07:02.279><c> uh</c><01:07:02.440><c> so</c><01:07:02.760><c> are</c><01:07:03.119><c> there</c><01:07:03.559><c> it's</c>\nsomething new yeah uh so are there it's\nsomething new yeah uh so are there it's an<01:07:03.799><c> active</c><01:07:04.039><c> area</c><01:07:04.279><c> of</c><01:07:04.400><c> research</c><01:07:04.960><c> but</c><01:07:05.240><c> any</c>\nan active area of research but any\nan active area of research but any thoughts<01:07:05.839><c> that</c><01:07:05.960><c> you</c><01:07:06.119><c> have</c><01:07:06.319><c> around</c><01:07:06.760><c> how</c><01:07:07.319><c> people</c>\nthoughts that you have around how people\nthoughts that you have around how people are<01:07:07.680><c> maybe</c><01:07:07.960><c> thinking</c><01:07:08.359><c> around</c><01:07:08.799><c> this</c><01:07:09.119><c> and</c><01:07:10.079><c> uh</c>\nare maybe thinking around this and uh\nare maybe thinking around this and uh better<01:07:10.599><c> ways</c><01:07:10.799><c> to</c><01:07:10.920><c> bootstrap</c><01:07:11.559><c> or</c><01:07:11.720><c> to</c><01:07:11.880><c> give</c><01:07:12.039><c> up</c>\nbetter ways to bootstrap or to give up\nbetter ways to bootstrap or to give up on<01:07:12.319><c> this</c><01:07:12.480><c> idea</c><01:07:12.799><c> and</c><01:07:13.240><c> and</c><01:07:13.400><c> realize</c><01:07:13.920><c> that</c><01:07:14.319><c> the</c>\non this idea and and realize that the\non this idea and and realize that the chart<01:07:14.720><c> shows</c><01:07:15.079><c> you</c><01:07:15.200><c> don't</c><01:07:15.400><c> need</c><01:07:15.599><c> that</c><01:07:15.760><c> many</c><01:07:15.920><c> so</c>\nchart shows you don't need that many so\nchart shows you don't need that many so just<01:07:16.359><c> get</c><01:07:16.520><c> humans</c><01:07:16.839><c> to</c><01:07:17.000><c> generate</c><01:07:17.400><c> 2,000</c><01:07:17.920><c> really</c>\njust get humans to generate 2,000 really\njust get humans to generate 2,000 really good<01:07:18.920><c> uh</c><01:07:19.400><c> yeah</c><01:07:20.160><c> so</c><01:07:20.359><c> that's</c><01:07:20.480><c> a</c><01:07:20.599><c> very</c><01:07:20.760><c> good</c>\ngood uh yeah so that's a very good\ngood uh yeah so that's a very good question<01:07:21.640><c> uh</c><01:07:21.839><c> so</c><01:07:22.039><c> for</c><01:07:22.240><c> the</c><01:07:22.400><c> data</c><01:07:22.720><c> stuff</c><01:07:23.000><c> so</c><01:07:23.200><c> I'm</c>\nquestion uh so for the data stuff so I'm\nquestion uh so for the data stuff so I'm saying<01:07:23.520><c> it's</c><01:07:23.640><c> not</c><01:07:23.799><c> that</c><01:07:23.920><c> important</c><01:07:24.200><c> for</c><01:07:24.359><c> sft</c>\nsaying it's not that important for sft\nsaying it's not that important for sft but<01:07:24.880><c> there</c><01:07:24.960><c> will</c><01:07:25.079><c> be</c><01:07:25.200><c> another</c><01:07:25.599><c> thing</c><01:07:25.720><c> we'll</c>\nbut there will be another thing we'll\nbut there will be another thing we'll talk<01:07:26.079><c> about</c><01:07:26.480><c> right</c><01:07:26.720><c> after</c><01:07:27.200><c> where</c><01:07:27.520><c> actually</c>\ntalk about right after where actually\ntalk about right after where actually data<01:07:28.480><c> does</c>\ndata does\ndata does matter<01:07:30.240><c> my</c><01:07:30.920><c> intuition</c><01:07:31.559><c> based</c><01:07:31.960><c> on</c><01:07:32.160><c> not</c><01:07:32.400><c> that</c>\nmatter my intuition based on not that\nmatter my intuition based on not that much<01:07:32.760><c> empirical</c><01:07:33.240><c> results</c><01:07:34.240><c> is</c><01:07:34.400><c> that</c><01:07:34.520><c> you</c><01:07:34.640><c> can</c>\nmuch empirical results is that you can\nmuch empirical results is that you can still<01:07:35.319><c> get</c><01:07:36.200><c> um</c><01:07:37.119><c> even</c><01:07:37.359><c> though</c><01:07:37.480><c> you</c><01:07:37.599><c> use</c><01:07:37.760><c> your</c>\nstill get um even though you use your\nstill get um even though you use your LMS<01:07:38.440><c> if</c><01:07:38.520><c> you</c><01:07:38.640><c> use</c><01:07:38.880><c> purely</c><01:07:39.200><c> LM</c><01:07:39.559><c> generated</c><01:07:40.039><c> text</c>\nLMS if you use purely LM generated text\nLMS if you use purely LM generated text and<01:07:40.880><c> you</c><01:07:41.000><c> do</c><01:07:41.240><c> that</c><01:07:41.400><c> for</c><01:07:41.599><c> like</c><01:07:41.799><c> three</c><01:07:42.039><c> four</c>\nand you do that for like three four\nand you do that for like three four generations<01:07:42.720><c> of</c><01:07:42.839><c> llms</c><01:07:43.279><c> I</c><01:07:43.400><c> agree</c><01:07:43.599><c> with</c><01:07:43.720><c> you</c>\ngenerations of llms I agree with you\ngenerations of llms I agree with you that<01:07:44.000><c> probably</c><01:07:44.200><c> you</c><01:07:44.279><c> won't</c><01:07:44.559><c> improve</c><01:07:45.000><c> much</c><01:07:46.000><c> but</c>\nthat probably you won't improve much but\nthat probably you won't improve much but for<01:07:46.279><c> me</c><01:07:46.440><c> what</c><01:07:46.559><c> is</c><01:07:46.720><c> important</c><01:07:47.039><c> is</c><01:07:47.160><c> how</c><01:07:47.240><c> do</c><01:07:47.319><c> you</c>\nfor me what is important is how do you\nfor me what is important is how do you use<01:07:47.760><c> like</c><01:07:47.920><c> human</c><01:07:48.200><c> in</c><01:07:48.319><c> the</c><01:07:48.480><c> loop</c><01:07:48.839><c> with</c><01:07:49.039><c> llms</c><01:07:49.960><c> not</c>\nuse like human in the loop with llms not\nuse like human in the loop with llms not purely<01:07:50.599><c> LMS</c><01:07:51.200><c> not</c><01:07:51.440><c> purely</c><01:07:52.279><c> uh</c><01:07:52.880><c> humans</c><01:07:53.240><c> but</c>\npurely LMS not purely uh humans but\npurely LMS not purely uh humans but maybe<01:07:53.640><c> what</c><01:07:53.720><c> you</c><01:07:53.799><c> can</c><01:07:53.920><c> do</c><01:07:54.079><c> is</c><01:07:54.240><c> just</c><01:07:54.440><c> have</c><01:07:54.760><c> the</c>\nmaybe what you can do is just have the\nmaybe what you can do is just have the model<01:07:55.520><c> generate</c><01:07:55.839><c> some</c><01:07:56.000><c> new</c><01:07:56.160><c> text</c><01:07:56.680><c> and</c><01:07:56.880><c> just</c><01:07:57.520><c> uh</c>\nmodel generate some new text and just uh\nmodel generate some new text and just uh humans<01:07:58.079><c> write</c><01:07:58.240><c> a</c><01:07:58.359><c> few</c><01:07:58.599><c> Edits</c><01:07:59.240><c> edits</c><01:07:59.520><c> are</c><01:07:59.720><c> much</c>\nhumans write a few Edits edits are much\nhumans write a few Edits edits are much faster<01:08:00.599><c> than</c><01:08:00.799><c> writing</c><01:08:01.079><c> the</c><01:08:01.240><c> entire</c><01:08:01.599><c> text</c><01:08:02.119><c> and</c>\nfaster than writing the entire text and\nfaster than writing the entire text and I<01:08:02.319><c> think</c><01:08:02.480><c> that</c><01:08:02.599><c> if</c><01:08:02.680><c> you</c><01:08:02.799><c> have</c><01:08:02.960><c> that</c><01:08:03.079><c> type</c><01:08:03.240><c> of</c>\nI think that if you have that type of\nI think that if you have that type of collaboration<01:08:04.400><c> then</c><01:08:04.680><c> from</c><01:08:04.960><c> like</c><01:08:05.119><c> kind</c><01:08:05.240><c> of</c><01:08:05.359><c> an</c>\ncollaboration then from like kind of an\ncollaboration then from like kind of an information<01:08:05.960><c> theoretical</c><01:08:06.440><c> point</c><01:08:06.599><c> of</c><01:08:06.720><c> view</c>\ninformation theoretical point of view\ninformation theoretical point of view you<01:08:07.200><c> still</c><01:08:07.559><c> get</c><01:08:08.000><c> additional</c><01:08:08.440><c> information</c><01:08:09.119><c> but</c>\nyou still get additional information but\nyou still get additional information but you<01:08:09.400><c> still</c><01:08:09.680><c> much</c><01:08:09.920><c> faster</c><01:08:10.279><c> than</c><01:08:10.400><c> if</c><01:08:10.520><c> you</c><01:08:10.680><c> use</c>\nyou still much faster than if you use\nyou still much faster than if you use humans<01:08:11.640><c> and</c><01:08:11.760><c> I</c><01:08:11.880><c> think</c><01:08:12.039><c> that</c><01:08:12.319><c> as</c><01:08:12.400><c> a</c><01:08:12.559><c> field</c><01:08:12.920><c> we'll</c>\nhumans and I think that as a field we'll\nhumans and I think that as a field we'll probably<01:08:13.440><c> move</c><01:08:13.720><c> towards</c><01:08:14.119><c> these</c><01:08:14.319><c> type</c><01:08:14.480><c> of</c>\nprobably move towards these type of\nprobably move towards these type of things<01:08:15.319><c> uh</c><01:08:15.440><c> which</c><01:08:15.640><c> is</c><01:08:16.640><c> um</c><01:08:16.839><c> really</c><01:08:17.120><c> just</c>\nthings uh which is um really just\nthings uh which is um really just finding<01:08:17.839><c> the</c><01:08:18.000><c> examples</c><01:08:18.759><c> that</c><01:08:18.880><c> are</c><01:08:19.080><c> important</c>\nfinding the examples that are important\nfinding the examples that are important and<01:08:19.839><c> and</c><01:08:20.239><c> asking</c><01:08:20.679><c> humans</c><01:08:21.159><c> it's</c><01:08:21.279><c> kind</c><01:08:21.400><c> of</c>\nand and asking humans it's kind of\nand and asking humans it's kind of active<01:08:21.759><c> learning</c><01:08:22.120><c> just</c><01:08:22.279><c> asking</c><01:08:22.560><c> humans</c>\nactive learning just asking humans\nactive learning just asking humans exactly<01:08:23.400><c> when</c><01:08:24.120><c> uh</c><01:08:24.239><c> you</c><01:08:24.400><c> need</c><01:08:24.600><c> to</c><01:08:25.040><c> to</c><01:08:25.239><c> get</c>\nexactly when uh you need to to get\nexactly when uh you need to to get inputs<01:08:28.319><c> yes</c><01:08:28.759><c> do</c><01:08:28.880><c> we</c><01:08:29.080><c> train</c><01:08:29.400><c> with</c><01:08:29.640><c> like</c><01:08:29.759><c> the</c>\ninputs yes do we train with like the\ninputs yes do we train with like the same<01:08:30.120><c> loss</c><01:08:30.400><c> function</c><01:08:30.880><c> the</c><01:08:31.000><c> same</c><01:08:31.400><c> like</c><01:08:31.640><c> General</c>\nsame loss function the same like General\nsame loss function the same like General training<01:08:32.359><c> algorithm</c><01:08:32.799><c> for</c><01:08:32.920><c> the</c><01:08:33.080><c> supervis</c>\ntraining algorithm for the supervis\ntraining algorithm for the supervis tuning<01:08:34.120><c> bit</c><01:08:34.359><c> as</c><01:08:34.480><c> we</c><01:08:34.600><c> do</c><01:08:34.759><c> for</c><01:08:35.040><c> the</c><01:08:35.239><c> for</c><01:08:35.400><c> the</c>\ntuning bit as we do for the for the\ntuning bit as we do for the for the pre-training<01:08:36.199><c> right</c><01:08:36.400><c> because</c><01:08:36.719><c> like</c><01:08:37.520><c> the</c>\npre-training right because like the\npre-training right because like the examples<01:08:38.080><c> you</c><01:08:38.239><c> showed</c><01:08:39.080><c> I</c><01:08:39.159><c> think</c><01:08:39.480><c> the</c><01:08:39.679><c> the</c>\nexamples you showed I think the the\nexamples you showed I think the the important<01:08:40.319><c> thing</c><01:08:40.679><c> of</c><01:08:41.480><c> the</c><01:08:42.480><c> good</c><01:08:42.640><c> examples</c><01:08:43.120><c> is</c>\nimportant thing of the good examples is\nimportant thing of the good examples is they're<01:08:43.719><c> like</c><01:08:43.880><c> supera</c><01:08:44.679><c> accurate</c><01:08:45.520><c> there's</c>\nthey're like supera accurate there's\nthey're like supera accurate there's these<01:08:46.000><c> more</c><01:08:46.679><c> complex</c><01:08:47.679><c> still</c><01:08:47.960><c> just</c><01:08:48.159><c> like</c><01:08:48.319><c> chain</c>\nthese more complex still just like chain\nthese more complex still just like chain same<01:08:49.400><c> so</c><01:08:49.600><c> that's</c><01:08:49.799><c> why</c><01:08:50.040><c> here</c><01:08:50.400><c> I</c><01:08:50.600><c> yeah</c><01:08:50.759><c> I</c><01:08:50.880><c> didn't</c>\nsame so that's why here I yeah I didn't\nsame so that's why here I yeah I didn't maybe<01:08:51.279><c> didn't</c><01:08:51.520><c> emphasize</c><01:08:52.040><c> enough</c><01:08:52.600><c> this</c><01:08:52.679><c> is</c>\nmaybe didn't emphasize enough this is\nmaybe didn't emphasize enough this is just<01:08:53.040><c> language</c><01:08:53.359><c> modeling</c><01:08:53.759><c> fine</c><01:08:54.000><c> tun</c><01:08:54.199><c> the</c><01:08:54.319><c> LM</c>\njust language modeling fine tun the LM\njust language modeling fine tun the LM with<01:08:54.719><c> language</c><01:08:55.000><c> model</c><01:08:55.279><c> on</c><01:08:55.560><c> the</c><01:08:55.719><c> desired</c>\nwith language model on the desired\nwith language model on the desired answers<01:08:56.679><c> so</c><01:08:56.839><c> this</c><01:08:56.960><c> is</c><01:08:57.159><c> literally</c><01:08:57.520><c> the</c><01:08:57.640><c> same</c>\nanswers so this is literally the same\nanswers so this is literally the same loss<01:08:58.679><c> um</c><01:08:59.279><c> it</c><01:08:59.440><c> will</c><01:08:59.600><c> be</c><01:08:59.839><c> different</c><01:09:00.719><c> in</c><01:09:00.880><c> two</c>\nloss um it will be different in two\nloss um it will be different in two seconds<01:09:01.880><c> but</c><01:09:02.120><c> the</c><01:09:02.359><c> first</c><01:09:02.640><c> step</c><01:09:02.839><c> of</c><01:09:03.000><c> sft</c><01:09:03.600><c> is</c>\nseconds but the first step of sft is\nseconds but the first step of sft is literally<01:09:04.159><c> the</c><01:09:04.279><c> same</c><01:09:04.520><c> loss</c><01:09:05.040><c> where</c><01:09:05.159><c> you</c><01:09:05.359><c> just</c>\nliterally the same loss where you just\nliterally the same loss where you just say<01:09:05.839><c> Okay</c><01:09:06.000><c> I</c><01:09:06.080><c> want</c><01:09:06.199><c> to</c><01:09:06.440><c> actually</c><01:09:06.719><c> specialize</c>\nsay Okay I want to actually specialize\nsay Okay I want to actually specialize on<01:09:07.359><c> that</c><01:09:07.520><c> type</c><01:09:07.679><c> of</c><01:09:07.839><c> data</c><01:09:08.319><c> so</c><01:09:08.520><c> there's</c><01:09:08.719><c> even</c><01:09:08.920><c> a</c>\non that type of data so there's even a\non that type of data so there's even a question<01:09:09.319><c> of</c><01:09:09.520><c> like</c><01:09:09.839><c> what</c><01:09:10.000><c> is</c><01:09:10.159><c> pre-training</c>\nquestion of like what is pre-training\nquestion of like what is pre-training what<01:09:10.799><c> is</c><01:09:10.920><c> post-training</c><01:09:11.480><c> because</c><01:09:11.640><c> in</c><01:09:11.759><c> reality</c>\nwhat is post-training because in reality\nwhat is post-training because in reality it's<01:09:12.120><c> just</c><01:09:12.239><c> like</c><01:09:12.359><c> a</c><01:09:12.480><c> different</c><01:09:12.759><c> data</c><01:09:13.040><c> that</c><01:09:13.159><c> you</c>\nit's just like a different data that you\nit's just like a different data that you use<01:09:13.759><c> the</c><01:09:13.880><c> reason</c><01:09:14.159><c> why</c><01:09:14.279><c> we</c><01:09:14.480><c> usually</c><01:09:14.719><c> call</c><01:09:14.880><c> it</c>\nuse the reason why we usually call it\nuse the reason why we usually call it post<01:09:15.239><c> training</c><01:09:15.560><c> is</c><01:09:15.640><c> that</c><01:09:15.799><c> the</c><01:09:15.880><c> way</c><01:09:16.000><c> we</c><01:09:16.120><c> collect</c>\npost training is that the way we collect\npost training is that the way we collect that<01:09:16.560><c> data</c><01:09:16.759><c> is</c><01:09:16.920><c> very</c>\nthat data is very\nthat data is very different<01:09:19.040><c> great</c><01:09:19.520><c> great</c><01:09:19.960><c> questions</c><01:09:20.960><c> uh</c><01:09:21.159><c> yes</c>\ndifferent great great questions uh yes\ndifferent great great questions uh yes maybe<01:09:22.279><c> it's</c><01:09:22.400><c> the</c><01:09:22.600><c> same</c><01:09:22.960><c> question</c><01:09:23.319><c> but</c><01:09:23.520><c> why</c>\nmaybe it's the same question but why\nmaybe it's the same question but why would<01:09:24.000><c> these</c><01:09:24.239><c> 2,000</c><01:09:24.920><c> examples</c><01:09:25.640><c> have</c><01:09:25.880><c> such</c><01:09:26.040><c> an</c>\nwould these 2,000 examples have such an\nwould these 2,000 examples have such an overweighted\noverweighted\noverweighted influence<01:09:29.040><c> you</c><01:09:29.679><c> tun</c><01:09:30.239><c> so</c><01:09:30.400><c> that's</c><01:09:30.600><c> why</c><01:09:30.839><c> we</c><01:09:31.400><c> uh</c>\ninfluence you tun so that's why we uh\ninfluence you tun so that's why we uh also<01:09:32.000><c> that's</c><01:09:32.159><c> another</c><01:09:32.400><c> reason</c><01:09:32.679><c> why</c><01:09:32.799><c> we</c><01:09:32.920><c> call</c>\nalso that's another reason why we call\nalso that's another reason why we call it<01:09:33.159><c> post</c><01:09:33.400><c> training</c><01:09:33.679><c> is</c><01:09:33.799><c> that</c><01:09:33.920><c> we</c><01:09:34.040><c> use</c>\nit post training is that we use\nit post training is that we use different<01:09:34.480><c> type</c><01:09:34.640><c> of</c><01:09:34.759><c> hyper</c><01:09:35.040><c> parameters</c><01:09:35.640><c> so</c>\ndifferent type of hyper parameters so\ndifferent type of hyper parameters so you<01:09:35.839><c> know</c><01:09:36.000><c> I</c><01:09:36.120><c> told</c><01:09:36.319><c> you</c><01:09:36.600><c> basically</c><01:09:36.920><c> at</c><01:09:37.000><c> the</c><01:09:37.080><c> end</c>\nyou know I told you basically at the end\nyou know I told you basically at the end of<01:09:37.279><c> pre</c><01:09:37.440><c> training</c><01:09:37.759><c> you</c><01:09:37.960><c> essentially</c><01:09:38.319><c> end</c><01:09:38.480><c> up</c>\nof pre training you essentially end up\nof pre training you essentially end up with<01:09:38.719><c> a</c><01:09:38.799><c> learning</c><01:09:39.120><c> rate</c><01:09:39.279><c> of</c><01:09:39.440><c> zero</c><01:09:40.239><c> and</c><01:09:40.400><c> here</c>\nwith a learning rate of zero and here\nwith a learning rate of zero and here you're<01:09:40.640><c> going</c><01:09:40.759><c> to</c><01:09:40.880><c> increase</c><01:09:41.199><c> your</c><01:09:41.359><c> learning</c>\nyou're going to increase your learning\nyou're going to increase your learning rate<01:09:42.080><c> so</c><01:09:42.279><c> like</c><01:09:42.400><c> 1</c><01:09:42.560><c> eus</c><01:09:43.000><c> 5</c><01:09:43.199><c> one</c><01:09:43.359><c> E</c><01:09:43.920><c> Yeah</c><01:09:44.319><c> and</c><01:09:44.560><c> and</c>\nrate so like 1 eus 5 one E Yeah and and\nrate so like 1 eus 5 one E Yeah and and so<01:09:45.839><c> um</c><01:09:46.279><c> the</c><01:09:46.480><c> weight</c><01:09:46.759><c> that</c><01:09:46.880><c> you</c><01:09:47.040><c> give</c><01:09:47.199><c> to</c><01:09:47.440><c> them</c>\nso um the weight that you give to them\nso um the weight that you give to them is<01:09:48.159><c> actually</c>\nis actually\nis actually different\ndifferent\ndifferent um<01:09:52.960><c> okay</c><01:09:53.960><c> uh</c><01:09:54.159><c> Second</c><01:09:54.480><c> Step</c><01:09:54.840><c> or</c><01:09:55.040><c> second</c><01:09:55.560><c> part</c><01:09:55.960><c> of</c>\num okay uh Second Step or second part of\num okay uh Second Step or second part of this<01:09:56.719><c> post</c><01:09:57.000><c> training</c><01:09:57.840><c> um</c><01:09:58.080><c> is</c><01:09:58.280><c> what</c><01:09:58.400><c> we</c><01:09:58.560><c> call</c>\nthis post training um is what we call\nthis post training um is what we call reinforcement<01:09:59.760><c> learning</c><01:10:00.040><c> from</c><01:10:00.280><c> Human</c>\nreinforcement learning from Human\nreinforcement learning from Human feedback<01:10:01.120><c> or</c><01:10:01.440><c> rhf</c><01:10:02.440><c> uh</c><01:10:02.560><c> some</c><01:10:02.760><c> of</c><01:10:02.880><c> you</c><01:10:03.040><c> might</c>\nfeedback or rhf uh some of you might\nfeedback or rhf uh some of you might have<01:10:03.440><c> heard</c><01:10:03.640><c> of</c><01:10:03.800><c> that</c><01:10:04.719><c> um</c><01:10:05.520><c> the</c><01:10:05.760><c> idea</c><01:10:06.120><c> is</c><01:10:06.239><c> that</c>\nhave heard of that um the idea is that\nhave heard of that um the idea is that sft<01:10:06.920><c> has</c><01:10:07.040><c> a</c><01:10:07.280><c> problem</c><01:10:07.960><c> namely</c><01:10:08.520><c> that</c><01:10:08.960><c> uh</c><01:10:09.040><c> you</c><01:10:09.159><c> do</c>\nsft has a problem namely that uh you do\nsft has a problem namely that uh you do behavioral<01:10:10.239><c> cloning</c><01:10:10.840><c> which</c><01:10:10.960><c> means</c><01:10:11.199><c> that</c><01:10:11.360><c> you</c>\nbehavioral cloning which means that you\nbehavioral cloning which means that you just<01:10:11.640><c> try</c><01:10:11.840><c> to</c><01:10:12.040><c> clone</c><01:10:12.760><c> what</c><01:10:12.960><c> the</c><01:10:13.159><c> humans</c><01:10:13.800><c> would</c>\njust try to clone what the humans would\njust try to clone what the humans would say<01:10:14.679><c> and</c><01:10:14.800><c> that</c><01:10:15.000><c> had</c><01:10:15.520><c> that</c><01:10:15.640><c> has</c><01:10:15.760><c> many</c><01:10:16.000><c> issues</c>\nsay and that had that has many issues\nsay and that had that has many issues one<01:10:16.840><c> of</c><01:10:17.040><c> them</c><01:10:17.239><c> is</c><01:10:17.360><c> that</c><01:10:17.480><c> you're</c><01:10:17.640><c> bound</c><01:10:17.920><c> by</c>\none of them is that you're bound by\none of them is that you're bound by human<01:10:18.520><c> abilities</c><01:10:19.520><c> so</c><01:10:20.280><c> if</c><01:10:21.239><c> um</c><01:10:22.239><c> like</c><01:10:22.520><c> humans</c>\nhuman abilities so if um like humans\nhuman abilities so if um like humans actually<01:10:24.560><c> humans</c><01:10:25.320><c> won't</c><01:10:25.600><c> generate</c><01:10:26.000><c> the</c>\nactually humans won't generate the\nactually humans won't generate the things<01:10:26.400><c> that</c><01:10:26.560><c> they</c><01:10:26.719><c> think</c><01:10:26.880><c> is</c><01:10:27.040><c> actually</c><01:10:27.280><c> the</c>\nthings that they think is actually the\nthings that they think is actually the best<01:10:27.560><c> thing</c><01:10:27.719><c> to</c><01:10:27.840><c> generate</c><01:10:28.600><c> so</c><01:10:28.920><c> if</c><01:10:29.040><c> you</c><01:10:29.280><c> ask</c><01:10:29.480><c> me</c>\nbest thing to generate so if you ask me\nbest thing to generate so if you ask me to<01:10:29.800><c> write</c><01:10:29.960><c> a</c><01:10:30.120><c> book</c><01:10:30.760><c> I</c><01:10:30.840><c> mean</c><01:10:31.000><c> I</c><01:10:31.080><c> can</c><01:10:31.199><c> definitely</c>\nto write a book I mean I can definitely\nto write a book I mean I can definitely enjoy<01:10:31.800><c> a</c><01:10:31.960><c> book</c><01:10:32.280><c> I</c><01:10:32.360><c> can</c><01:10:32.520><c> probably</c><01:10:32.760><c> say</c><01:10:32.960><c> one</c><01:10:33.120><c> book</c>\nenjoy a book I can probably say one book\nenjoy a book I can probably say one book is<01:10:33.440><c> better</c><01:10:33.640><c> than</c><01:10:33.800><c> another</c><01:10:34.640><c> but</c><01:10:34.760><c> I'm</c>\nis better than another but I'm\nis better than another but I'm definitely<01:10:35.159><c> not</c><01:10:35.280><c> going</c><01:10:35.360><c> to</c><01:10:35.480><c> be</c><01:10:35.640><c> as</c><01:10:35.760><c> good</c><01:10:35.920><c> as</c>\ndefinitely not going to be as good as\ndefinitely not going to be as good as writing<01:10:36.360><c> the</c><01:10:36.480><c> book</c><01:10:36.640><c> that</c><01:10:36.760><c> I</c><01:10:36.880><c> want</c><01:10:37.040><c> to</c><01:10:37.239><c> read</c><01:10:38.000><c> uh</c>\nwriting the book that I want to read uh\nwriting the book that I want to read uh so<01:10:38.239><c> you're</c><01:10:38.400><c> going</c><01:10:38.480><c> to</c><01:10:38.600><c> be</c><01:10:38.719><c> bound</c><01:10:39.040><c> by</c><01:10:39.159><c> the</c><01:10:39.280><c> human</c>\nso you're going to be bound by the human\nso you're going to be bound by the human ability<01:10:39.880><c> to</c><01:10:40.000><c> generate</c><01:10:40.400><c> things</c><01:10:40.719><c> even</c><01:10:40.960><c> though</c>\nability to generate things even though\nability to generate things even though the<01:10:41.280><c> humans</c><01:10:41.560><c> might</c><01:10:41.719><c> be</c><01:10:41.840><c> better</c><01:10:42.080><c> at</c>\nthe humans might be better at\nthe humans might be better at distinguishing<01:10:42.880><c> between</c><01:10:43.199><c> things</c><01:10:43.800><c> that's</c><01:10:44.000><c> one</c>\ndistinguishing between things that's one\ndistinguishing between things that's one issue<01:10:44.880><c> issue</c><01:10:45.120><c> number</c><01:10:45.360><c> two</c><01:10:46.280><c> uh</c><01:10:46.400><c> I</c><01:10:46.520><c> find</c><01:10:46.679><c> that</c>\nissue issue number two uh I find that\nissue issue number two uh I find that actually<01:10:47.120><c> pretty</c><01:10:47.320><c> interesting</c><01:10:47.800><c> is</c><01:10:48.000><c> that</c><01:10:48.600><c> it</c>\nactually pretty interesting is that it\nactually pretty interesting is that it might<01:10:49.159><c> if</c><01:10:49.280><c> you</c><01:10:49.400><c> ever</c><01:10:49.600><c> heard</c><01:10:49.760><c> of</c><01:10:49.920><c> the</c><01:10:50.000><c> word</c>\nmight if you ever heard of the word\nmight if you ever heard of the word hallucination<01:10:50.960><c> so</c><01:10:51.159><c> this</c><01:10:51.239><c> is</c><01:10:51.480><c> llms</c><01:10:52.080><c> generating</c>\nhallucination so this is llms generating\nhallucination so this is llms generating F<01:10:53.440><c> like</c><01:10:53.840><c> false</c><01:10:54.239><c> information</c>\nF like false information\nF like false information hallucination<01:10:57.159><c> might</c><01:10:57.480><c> these</c><01:10:57.679><c> people</c><01:10:57.960><c> have</c><01:10:58.520><c> um</c>\nhallucination might these people have um\nhallucination might these people have um hypothesized<01:10:59.480><c> that</c><01:10:59.679><c> that</c><01:10:59.840><c> can</c><01:11:00.080><c> come</c><01:11:00.320><c> from</c><01:11:00.560><c> the</c>\nhypothesized that that can come from the\nhypothesized that that can come from the supervised<01:11:01.199><c> fine</c><01:11:01.400><c> tuning</c><01:11:02.120><c> even</c><01:11:02.360><c> if</c><01:11:02.480><c> you</c><01:11:02.600><c> do</c>\nsupervised fine tuning even if you do\nsupervised fine tuning even if you do supervised<01:11:03.320><c> fine</c><01:11:03.520><c> tuning</c><01:11:04.239><c> on</c><01:11:04.840><c> data</c><01:11:05.159><c> that</c><01:11:05.280><c> is</c>\nsupervised fine tuning on data that is\nsupervised fine tuning on data that is correct<01:11:06.320><c> and</c><01:11:06.440><c> the</c><01:11:06.600><c> reason</c><01:11:06.960><c> why</c><01:11:07.159><c> that</c><01:11:07.360><c> is</c><01:11:08.000><c> is</c>\ncorrect and the reason why that is is\ncorrect and the reason why that is is that<01:11:08.600><c> if</c><01:11:09.440><c> uh</c><01:11:09.600><c> given</c><01:11:09.920><c> I</c><01:11:10.040><c> told</c><01:11:10.239><c> you</c><01:11:10.360><c> that</c>\nthat if uh given I told you that\nthat if uh given I told you that basically<01:11:10.960><c> sftt</c><01:11:11.640><c> is</c><01:11:11.840><c> with</c><01:11:12.199><c> very</c><01:11:12.400><c> little</c><01:11:12.679><c> data</c>\nbasically sftt is with very little data\nbasically sftt is with very little data and<01:11:13.440><c> it's</c><01:11:13.679><c> with</c><01:11:13.880><c> data</c><01:11:14.719><c> that</c><01:11:14.880><c> doesn't</c><01:11:15.320><c> the</c>\nand it's with data that doesn't the\nand it's with data that doesn't the model<01:11:15.840><c> doesn't</c><01:11:16.080><c> learn</c><01:11:16.360><c> anything</c><01:11:16.719><c> new</c><01:11:17.480><c> so</c><01:11:17.760><c> what</c>\nmodel doesn't learn anything new so what\nmodel doesn't learn anything new so what if<01:11:18.239><c> the</c><01:11:18.440><c> human</c><01:11:18.800><c> gives</c><01:11:19.000><c> an</c><01:11:19.480><c> answer</c><01:11:20.480><c> that</c><01:11:20.640><c> the</c>\nif the human gives an answer that the\nif the human gives an answer that the model<01:11:21.159><c> didn't</c><01:11:21.440><c> know</c><01:11:22.000><c> was</c><01:11:22.480><c> true</c><01:11:23.480><c> from</c><01:11:23.679><c> the</c>\nmodel didn't know was true from the\nmodel didn't know was true from the model<01:11:24.199><c> perspective</c><01:11:24.960><c> you</c><01:11:25.400><c> the</c><01:11:25.560><c> human</c>\nmodel perspective you the human\nmodel perspective you the human basically<01:11:26.199><c> is</c><01:11:26.320><c> telling</c><01:11:26.760><c> the</c><01:11:27.000><c> the</c><01:11:27.120><c> model</c><01:11:28.120><c> uh</c>\nbasically is telling the the model uh\nbasically is telling the the model uh generate<01:11:28.880><c> this</c><01:11:29.080><c> thing</c><01:11:29.360><c> that</c><01:11:29.520><c> seems</c><01:11:30.040><c> plausible</c>\ngenerate this thing that seems plausible\ngenerate this thing that seems plausible but<01:11:31.280><c> actually</c><01:11:31.520><c> have</c><01:11:31.640><c> no</c><01:11:31.800><c> idea</c><01:11:32.080><c> if</c><01:11:32.159><c> it's</c><01:11:32.360><c> true</c>\nbut actually have no idea if it's true\nbut actually have no idea if it's true or<01:11:32.719><c> not</c><01:11:33.679><c> um</c><01:11:34.520><c> so</c><01:11:34.800><c> just</c><01:11:34.920><c> to</c><01:11:35.080><c> give</c><01:11:35.199><c> you</c><01:11:35.280><c> a</c><01:11:35.400><c> very</c>\nor not um so just to give you a very\nor not um so just to give you a very concrete<01:11:35.960><c> example</c><01:11:36.560><c> if</c><01:11:36.679><c> we</c><01:11:36.840><c> go</c><01:11:37.000><c> back</c><01:11:37.159><c> to</c><01:11:37.360><c> this</c>\nconcrete example if we go back to this\nconcrete example if we go back to this uh<01:11:37.880><c> monopsony</c><01:11:38.560><c> example</c><01:11:39.000><c> can</c><01:11:39.120><c> you</c><01:11:39.280><c> write</c><01:11:39.480><c> blah</c>\nuh monopsony example can you write blah\nuh monopsony example can you write blah blah<01:11:39.840><c> blah</c><01:11:40.320><c> about</c><01:11:40.639><c> monopsony</c><01:11:41.639><c> uh</c><01:11:41.800><c> imagine</c>\nblah blah about monopsony uh imagine\nblah blah about monopsony uh imagine that<01:11:42.239><c> a</c><01:11:42.440><c> human</c><01:11:43.159><c> uh</c><01:11:43.320><c> wrote</c><01:11:43.560><c> a</c><01:11:43.760><c> reference</c><01:11:44.440><c> on</c>\nthat a human uh wrote a reference on\nthat a human uh wrote a reference on this<01:11:44.960><c> type</c><01:11:45.159><c> of</c><01:11:45.320><c> book</c><01:11:46.239><c> um</c><01:11:46.679><c> and</c><01:11:46.920><c> that</c><01:11:47.080><c> book</c><01:11:47.280><c> might</c>\nthis type of book um and that book might\nthis type of book um and that book might exist<01:11:47.840><c> that</c><01:11:47.960><c> might</c><01:11:48.080><c> be</c><01:11:48.199><c> a</c><01:11:48.320><c> correct</c><01:11:48.679><c> reference</c>\nexist that might be a correct reference\nexist that might be a correct reference but<01:11:49.600><c> what</c><01:11:49.719><c> if</c><01:11:49.840><c> the</c><01:11:49.960><c> llm</c><01:11:50.440><c> never</c><01:11:50.679><c> saw</c><01:11:51.080><c> this</c>\nbut what if the llm never saw this\nbut what if the llm never saw this reference<01:11:51.679><c> during</c><01:11:51.960><c> pre-training</c><01:11:52.760><c> then</c><01:11:52.880><c> it</c>\nreference during pre-training then it\nreference during pre-training then it doesn't<01:11:53.280><c> know</c><01:11:53.480><c> that</c><01:11:53.600><c> it's</c><01:11:53.679><c> a</c><01:11:53.800><c> correct</c>\ndoesn't know that it's a correct\ndoesn't know that it's a correct reference<01:11:54.639><c> so</c><01:11:54.800><c> really</c><01:11:55.000><c> what</c><01:11:55.080><c> you</c><01:11:55.440><c> tell</c><01:11:55.639><c> the</c>\nreference so really what you tell the\nreference so really what you tell the model<01:11:56.400><c> is</c><01:11:56.600><c> to</c><01:11:57.000><c> generate</c><01:11:57.400><c> or</c><01:11:57.639><c> make</c><01:11:57.840><c> up</c><01:11:58.040><c> some</c>\nmodel is to generate or make up some\nmodel is to generate or make up some plausibly<01:11:58.760><c> sounding</c><01:11:59.520><c> reference</c><01:12:00.520><c> um</c><01:12:01.159><c> rather</c>\nplausibly sounding reference um rather\nplausibly sounding reference um rather than<01:12:01.760><c> actually</c><01:12:02.679><c> tell</c><01:12:02.920><c> the</c><01:12:03.080><c> real</c><01:12:03.360><c> reference</c>\nthan actually tell the real reference\nthan actually tell the real reference that<01:12:03.800><c> it</c><01:12:03.920><c> saw</c><01:12:04.120><c> during</c><01:12:04.440><c> pre-training</c><01:12:05.440><c> uh</c><01:12:05.600><c> so</c>\nthat it saw during pre-training uh so\nthat it saw during pre-training uh so hallucination<01:12:07.040><c> might</c><01:12:07.880><c> be</c><01:12:08.840><c> um</c><01:12:09.600><c> uh</c><01:12:09.920><c> a</c><01:12:10.159><c> re</c><01:12:10.480><c> like</c>\nhallucination might be um uh a re like\nhallucination might be um uh a re like might<01:12:10.840><c> be</c><01:12:11.000><c> caused</c><01:12:11.400><c> by</c><01:12:11.560><c> this</c><01:12:11.719><c> sft</c><01:12:12.520><c> that's</c>\nmight be caused by this sft that's\nmight be caused by this sft that's problem<01:12:13.239><c> number</c><01:12:13.440><c> two</c><01:12:14.199><c> does</c><01:12:14.360><c> that</c><01:12:14.480><c> all</c><01:12:14.639><c> make</c>\nproblem number two does that all make\nproblem number two does that all make sense<01:12:15.800><c> great</c><01:12:16.560><c> problem</c><01:12:16.840><c> number</c><01:12:17.080><c> three</c><01:12:17.639><c> price</c>\nsense great problem number three price\nsense great problem number three price generating<01:12:19.000><c> the</c><01:12:19.239><c> ideal</c><01:12:19.800><c> answers</c><01:12:20.800><c> is</c><01:12:21.040><c> very</c>\ngenerating the ideal answers is very\ngenerating the ideal answers is very pricey<01:12:21.679><c> and</c><01:12:21.800><c> that</c><01:12:21.960><c> comes</c><01:12:22.199><c> back</c><01:12:22.320><c> to</c><01:12:22.440><c> your</c>\npricey and that comes back to your\npricey and that comes back to your question<01:12:23.440><c> um</c><01:12:23.719><c> of</c><01:12:23.960><c> like</c><01:12:24.400><c> humans</c><01:12:24.679><c> writing</c>\nquestion um of like humans writing\nquestion um of like humans writing answer<01:12:26.000><c> is</c><01:12:26.199><c> actually</c><01:12:26.440><c> pretty</c>\nanswer is actually pretty\nanswer is actually pretty expensive<01:12:28.320><c> um</c><01:12:28.520><c> so</c><01:12:28.679><c> that's</c><01:12:28.840><c> where</c><01:12:29.000><c> rhf</c><01:12:29.600><c> comes</c>\nexpensive um so that's where rhf comes\nexpensive um so that's where rhf comes in<01:12:30.360><c> the</c><01:12:30.520><c> idea</c><01:12:30.840><c> is</c><01:12:31.000><c> that</c><01:12:31.239><c> instead</c><01:12:31.600><c> of</c><01:12:31.800><c> cloning</c>\nin the idea is that instead of cloning\nin the idea is that instead of cloning the<01:12:32.440><c> behaviors</c><01:12:33.360><c> of</c><01:12:33.560><c> humans</c><01:12:34.159><c> we're</c><01:12:34.320><c> going</c><01:12:34.400><c> to</c>\nthe behaviors of humans we're going to\nthe behaviors of humans we're going to maximize<01:12:35.320><c> human</c><01:12:35.920><c> preference</c><01:12:36.920><c> um</c><01:12:37.280><c> and</c><01:12:37.400><c> the</c><01:12:37.520><c> way</c>\nmaximize human preference um and the way\nmaximize human preference um and the way we're<01:12:37.800><c> going</c><01:12:37.880><c> to</c><01:12:38.040><c> do</c><01:12:38.239><c> that</c><01:12:38.440><c> so</c><01:12:38.639><c> the</c><01:12:38.800><c> pipeline</c>\nwe're going to do that so the pipeline\nwe're going to do that so the pipeline is<01:12:39.800><c> that</c><01:12:40.000><c> for</c><01:12:40.639><c> a</c><01:12:40.840><c> certain</c><01:12:41.320><c> for</c><01:12:41.480><c> every</c>\nis that for a certain for every\nis that for a certain for every instruction<01:12:42.280><c> you're</c><01:12:42.400><c> going</c><01:12:42.520><c> to</c><01:12:42.719><c> ask</c><01:12:42.880><c> a</c><01:12:43.040><c> model</c>\ninstruction you're going to ask a model\ninstruction you're going to ask a model to<01:12:43.440><c> generate</c><01:12:43.920><c> two</c><01:12:44.960><c> answers</c><01:12:45.960><c> um</c><01:12:46.520><c> and</c><01:12:46.719><c> usually</c>\nto generate two answers um and usually\nto generate two answers um and usually use<01:12:47.159><c> a</c><01:12:47.400><c> pretty</c><01:12:47.639><c> good</c><01:12:47.800><c> model</c><01:12:48.080><c> so</c><01:12:48.199><c> you</c><01:12:48.280><c> usually</c>\nuse a pretty good model so you usually\nuse a pretty good model so you usually don't<01:12:48.760><c> use</c><01:12:48.920><c> an</c><01:12:49.080><c> LM</c><01:12:49.480><c> here</c><01:12:49.639><c> you</c><01:12:49.760><c> use</c><01:12:49.960><c> a</c><01:12:51.040><c> sft</c><01:12:52.040><c> uh</c>\ndon't use an LM here you use a sft uh\ndon't use an LM here you use a sft uh fine<01:12:52.400><c> tune</c><01:12:52.800><c> you</c><01:12:52.880><c> use</c><01:12:53.040><c> a</c><01:12:53.199><c> fine</c><01:12:53.440><c> tuned</c><01:12:53.760><c> llm</c>\nfine tune you use a fine tuned llm\nfine tune you use a fine tuned llm already<01:12:54.760><c> to</c><01:12:54.920><c> give</c><01:12:55.239><c> like</c><01:12:55.520><c> pretty</c><01:12:55.760><c> good</c><01:12:56.239><c> answers</c>\nalready to give like pretty good answers\nalready to give like pretty good answers and<01:12:57.400><c> then</c><01:12:57.560><c> you</c><01:12:57.840><c> ask</c><01:12:58.239><c> labelers</c><01:12:59.239><c> which</c><01:12:59.400><c> of</c><01:12:59.600><c> these</c>\nand then you ask labelers which of these\nand then you ask labelers which of these two<01:13:00.000><c> answers</c><01:13:00.360><c> was</c><01:13:00.560><c> better</c><01:13:01.199><c> so</c><01:13:01.440><c> select</c><01:13:01.760><c> the</c>\ntwo answers was better so select the\ntwo answers was better so select the preferred<01:13:02.280><c> one</c><01:13:03.000><c> and</c><01:13:03.360><c> then</c><01:13:03.840><c> with</c><01:13:04.080><c> different</c>\npreferred one and then with different\npreferred one and then with different type<01:13:04.520><c> of</c><01:13:04.639><c> algorithms</c><01:13:05.120><c> we're</c><01:13:05.239><c> going</c><01:13:05.320><c> to</c><01:13:05.400><c> talk</c>\ntype of algorithms we're going to talk\ntype of algorithms we're going to talk about<01:13:05.719><c> the</c><01:13:05.800><c> algorithms</c><01:13:06.800><c> um</c><01:13:07.000><c> you</c><01:13:07.159><c> just</c>\nabout the algorithms um you just\nabout the algorithms um you just fine-tune<01:13:07.840><c> the</c><01:13:07.960><c> model</c><01:13:08.280><c> to</c><01:13:08.440><c> generate</c><01:13:08.840><c> more</c><01:13:09.000><c> of</c>\nfine-tune the model to generate more of\nfine-tune the model to generate more of the<01:13:09.239><c> green</c><01:13:09.560><c> thing</c><01:13:09.960><c> than</c><01:13:10.120><c> the</c><01:13:10.280><c> red</c><01:13:10.480><c> thing</c><01:13:10.800><c> so</c>\nthe green thing than the red thing so\nthe green thing than the red thing so more<01:13:11.080><c> of</c><01:13:11.239><c> the</c><01:13:11.360><c> good</c><01:13:11.679><c> stuff</c><01:13:12.679><c> uh</c><01:13:12.840><c> so</c><01:13:13.000><c> now</c><01:13:13.159><c> the</c>\nmore of the good stuff uh so now the\nmore of the good stuff uh so now the question<01:13:13.520><c> is</c><01:13:13.760><c> how</c><01:13:14.000><c> and</c><01:13:14.080><c> we're</c><01:13:14.199><c> going</c><01:13:14.280><c> to</c><01:13:14.400><c> talk</c>\nquestion is how and we're going to talk\nquestion is how and we're going to talk about<01:13:14.760><c> that</c><01:13:14.960><c> right</c>\nabout that right\nabout that right now<01:13:17.360><c> so</c><01:13:17.800><c> there</c><01:13:17.960><c> are</c><01:13:18.400><c> two</c><01:13:18.840><c> ways</c><01:13:19.080><c> that</c><01:13:19.199><c> we're</c>\nnow so there are two ways that we're\nnow so there are two ways that we're going<01:13:19.400><c> to</c><01:13:19.520><c> talk</c><01:13:19.719><c> about</c><01:13:20.000><c> and</c><01:13:20.159><c> two</c><01:13:20.360><c> that</c><01:13:20.480><c> are</c>\ngoing to talk about and two that are\ngoing to talk about and two that are mainly<01:13:21.159><c> used</c><01:13:21.360><c> in</c><01:13:21.480><c> the</c><01:13:21.600><c> community</c><01:13:22.600><c> um</c><01:13:23.360><c> the</c>\nmainly used in the community um the\nmainly used in the community um the first<01:13:23.760><c> one</c><01:13:24.199><c> is</c><01:13:24.440><c> simply</c><01:13:24.719><c> the</c><01:13:24.840><c> idea</c><01:13:25.040><c> of</c><01:13:25.120><c> of</c><01:13:25.239><c> using</c>\nfirst one is simply the idea of of using\nfirst one is simply the idea of of using reinforcement<01:13:25.960><c> learning</c><01:13:26.360><c> so</c><01:13:26.520><c> hopefully</c><01:13:26.880><c> you</c>\nreinforcement learning so hopefully you\nreinforcement learning so hopefully you all<01:13:27.120><c> know</c><01:13:27.280><c> what</c><01:13:27.400><c> reinforcement</c><01:13:27.880><c> learning</c><01:13:28.239><c> is</c>\nall know what reinforcement learning is\nall know what reinforcement learning is now<01:13:29.679><c> um</c><01:13:30.320><c> so</c><01:13:30.920><c> when</c><01:13:31.120><c> you</c><01:13:31.960><c> think</c><01:13:32.199><c> about</c><01:13:32.360><c> using</c>\nnow um so when you think about using\nnow um so when you think about using reinforcement<01:13:33.120><c> learning</c><01:13:33.400><c> one</c><01:13:33.560><c> important</c>\nreinforcement learning one important\nreinforcement learning one important question<01:13:34.080><c> is</c><01:13:34.239><c> like</c><01:13:34.400><c> what</c><01:13:34.520><c> is</c><01:13:34.639><c> the</c><01:13:34.760><c> reward</c><01:13:35.080><c> that</c>\nquestion is like what is the reward that\nquestion is like what is the reward that we're<01:13:35.400><c> optimizing</c><01:13:36.400><c> uh</c><01:13:36.520><c> so</c><01:13:36.800><c> in</c><01:13:36.960><c> this</c><01:13:37.120><c> case</c>\nwe're optimizing uh so in this case\nwe're optimizing uh so in this case there<01:13:37.440><c> are</c><01:13:37.840><c> really</c><01:13:38.239><c> two</c><01:13:38.440><c> options</c><01:13:38.760><c> that</c><01:13:38.880><c> I</c>\nthere are really two options that I\nthere are really two options that I could<01:13:39.120><c> think</c><01:13:39.320><c> about</c><01:13:39.840><c> the</c><01:13:39.960><c> first</c><01:13:40.159><c> one</c><01:13:40.360><c> you</c>\ncould think about the first one you\ncould think about the first one you could<01:13:40.639><c> just</c><01:13:40.800><c> say</c><01:13:41.440><c> I'm</c><01:13:41.560><c> going</c><01:13:41.679><c> to</c><01:13:41.840><c> compare</c><01:13:42.199><c> the</c>\ncould just say I'm going to compare the\ncould just say I'm going to compare the output<01:13:42.719><c> generated</c><01:13:43.120><c> by</c><01:13:43.239><c> some</c><01:13:43.440><c> baseline</c><01:13:44.080><c> the</c>\noutput generated by some baseline the\noutput generated by some baseline the output<01:13:44.600><c> generated</c><01:13:45.040><c> by</c><01:13:45.199><c> my</c><01:13:45.360><c> model</c><01:13:46.320><c> U</c><01:13:46.440><c> and</c><01:13:46.560><c> I'm</c>\noutput generated by my model U and I'm\noutput generated by my model U and I'm just<01:13:46.800><c> going</c><01:13:46.880><c> to</c><01:13:47.040><c> ask</c><01:13:47.239><c> the</c><01:13:47.400><c> human</c><01:13:48.199><c> to</c><01:13:48.480><c> say</c><01:13:48.800><c> which</c>\njust going to ask the human to say which\njust going to ask the human to say which one<01:13:49.159><c> is</c><01:13:49.280><c> better</c><01:13:49.600><c> and</c><01:13:49.719><c> I'm</c><01:13:50.120><c> going</c><01:13:50.199><c> to</c><01:13:50.400><c> use</c><01:13:50.960><c> this</c>\none is better and I'm going to use this\none is better and I'm going to use this as<01:13:51.199><c> a</c><01:13:51.360><c> reward</c><01:13:51.760><c> so</c><01:13:51.920><c> if</c><01:13:52.040><c> I'm</c><01:13:52.199><c> better</c><01:13:52.440><c> than</c><01:13:52.560><c> the</c>\nas a reward so if I'm better than the\nas a reward so if I'm better than the Baseline<01:13:53.360><c> this</c><01:13:53.480><c> is</c><01:13:53.560><c> a</c><01:13:53.760><c> plus</c><01:13:53.960><c> one</c><01:13:54.239><c> if</c><01:13:54.360><c> not</c><01:13:54.480><c> it's</c>\nBaseline this is a plus one if not it's\nBaseline this is a plus one if not it's a<01:13:54.679><c> minus</c><01:13:54.920><c> one</c><01:13:55.159><c> one</c><01:13:55.800><c> uh</c><01:13:55.880><c> so</c><01:13:56.080><c> now</c><01:13:56.239><c> it's</c><01:13:56.520><c> binary</c>\na minus one one uh so now it's binary\na minus one one uh so now it's binary reward<01:13:57.480><c> the</c><01:13:57.639><c> problem</c><01:13:57.840><c> with</c><01:13:58.000><c> binary</c><01:13:58.320><c> reward</c><01:13:58.600><c> is</c>\nreward the problem with binary reward is\nreward the problem with binary reward is that<01:13:58.840><c> it's</c><01:13:59.000><c> very</c><01:13:59.159><c> sparse</c><01:13:59.840><c> and</c><01:13:59.920><c> you</c><01:14:00.000><c> don't</c><01:14:00.199><c> get</c>\nthat it's very sparse and you don't get\nthat it's very sparse and you don't get much<01:14:00.560><c> information</c><01:14:00.960><c> out</c><01:14:01.120><c> of</c><01:14:01.280><c> it</c><01:14:01.840><c> uh</c><01:14:02.040><c> like</c><01:14:02.320><c> maybe</c>\nmuch information out of it uh like maybe\nmuch information out of it uh like maybe your<01:14:02.800><c> answer</c><01:14:03.239><c> was</c><01:14:03.400><c> slightly</c><01:14:03.800><c> better</c><01:14:04.480><c> maybe</c><01:14:04.719><c> it</c>\nyour answer was slightly better maybe it\nyour answer was slightly better maybe it was<01:14:05.080><c> like</c><01:14:05.560><c> way</c><01:14:05.880><c> better</c><01:14:06.480><c> and</c><01:14:06.560><c> you</c><01:14:06.679><c> don't</c><01:14:06.880><c> really</c>\nwas like way better and you don't really\nwas like way better and you don't really know<01:14:07.719><c> from</c><01:14:08.120><c> this</c><01:14:08.679><c> um</c><01:14:09.199><c> how</c><01:14:09.360><c> much</c><01:14:09.560><c> better</c><01:14:09.800><c> it</c><01:14:10.159><c> was</c>\nknow from this um how much better it was\nknow from this um how much better it was so<01:14:11.320><c> option</c><01:14:11.600><c> two</c><01:14:12.199><c> is</c><01:14:12.320><c> that</c><01:14:12.480><c> you</c><01:14:12.560><c> can</c><01:14:12.679><c> train</c><01:14:13.040><c> what</c>\nso option two is that you can train what\nso option two is that you can train what we<01:14:13.280><c> call</c><01:14:13.400><c> a</c><01:14:13.560><c> reward</c><01:14:13.920><c> model</c><01:14:14.360><c> which</c><01:14:14.480><c> is</c><01:14:14.639><c> simply</c><01:14:15.120><c> a</c>\nwe call a reward model which is simply a\nwe call a reward model which is simply a classifier<01:14:16.600><c> uh</c><01:14:16.760><c> so</c><01:14:16.920><c> you</c><01:14:17.040><c> use</c><01:14:17.280><c> machine</c>\nclassifier uh so you use machine\nclassifier uh so you use machine learning<01:14:18.120><c> to</c><01:14:18.639><c> to</c><01:14:18.920><c> classify</c><01:14:19.920><c> how</c><01:14:20.199><c> much</c><01:14:20.480><c> better</c>\nlearning to to classify how much better\nlearning to to classify how much better uh<01:14:21.800><c> two</c><01:14:22.480><c> outputs</c><01:14:23.000><c> are</c><01:14:23.639><c> from</c><01:14:23.880><c> the</c><01:14:24.080><c> preference</c>\nuh two outputs are from the preference\nuh two outputs are from the preference from<01:14:24.760><c> the</c><01:14:25.080><c> perspective</c><01:14:25.440><c> of</c><01:14:25.560><c> the</c><01:14:25.679><c> human</c><01:14:26.679><c> um</c><01:14:27.159><c> so</c>\nfrom the perspective of the human um so\nfrom the perspective of the human um so this<01:14:27.800><c> is</c><01:14:27.880><c> a</c><01:14:27.960><c> little</c><01:14:28.120><c> bit</c><01:14:28.280><c> meta</c><01:14:28.920><c> but</c><01:14:29.040><c> what</c><01:14:29.159><c> you</c>\nthis is a little bit meta but what you\nthis is a little bit meta but what you basically<01:14:29.679><c> do</c><01:14:29.920><c> is</c><01:14:30.000><c> that</c><01:14:30.159><c> you</c><01:14:30.239><c> train</c><01:14:31.080><c> uh</c><01:14:31.320><c> you</c>\nbasically do is that you train uh you\nbasically do is that you train uh you take<01:14:31.840><c> um</c><01:14:32.840><c> a</c><01:14:32.960><c> reward</c><01:14:33.280><c> model</c><01:14:33.560><c> R</c><01:14:33.880><c> which</c><01:14:34.000><c> is</c><01:14:34.120><c> a</c><01:14:34.639><c> uh</c>\ntake um a reward model R which is a uh\ntake um a reward model R which is a uh just<01:14:35.040><c> a</c><01:14:35.280><c> large</c><01:14:36.080><c> also</c><01:14:36.320><c> a</c><01:14:36.520><c> large</c><01:14:37.159><c> um</c><01:14:37.880><c> a</c><01:14:38.000><c> large</c>\njust a large also a large um a large\njust a large also a large um a large classifier<01:14:39.440><c> and</c><01:14:39.560><c> you</c><01:14:39.760><c> basically</c><01:14:40.280><c> ask</c><01:14:40.639><c> this</c>\nclassifier and you basically ask this\nclassifier and you basically ask this reward<01:14:41.159><c> model</c><01:14:41.480><c> you</c><01:14:41.600><c> give</c><01:14:41.760><c> it</c><01:14:41.960><c> the</c><01:14:42.159><c> input</c><01:14:42.679><c> and</c>\nreward model you give it the input and\nreward model you give it the input and the<01:14:43.080><c> actual</c><01:14:43.400><c> output</c><01:14:43.800><c> that</c><01:14:43.920><c> you</c><01:14:44.040><c> have</c><01:14:44.280><c> one</c><01:14:44.400><c> of</c>\nthe actual output that you have one of\nthe actual output that you have one of the<01:14:44.639><c> two</c><01:14:44.880><c> outputs</c><01:14:45.880><c> uh</c><01:14:46.040><c> and</c><01:14:46.120><c> you</c><01:14:46.360><c> just</c><01:14:47.040><c> um</c>\nthe two outputs uh and you just um\nthe two outputs uh and you just um exponentiate<01:14:48.159><c> that</c><01:14:48.320><c> so</c><01:14:48.480><c> that's</c><01:14:48.639><c> the</c><01:14:48.760><c> soft</c><01:14:49.040><c> Max</c>\nexponentiate that so that's the soft Max\nexponentiate that so that's the soft Max law<01:14:49.679><c> that</c><01:14:49.760><c> you</c><01:14:49.880><c> all</c><01:14:50.040><c> know</c><01:14:50.239><c> about</c><01:14:50.880><c> and</c><01:14:51.080><c> now</c><01:14:51.239><c> you</c>\nlaw that you all know about and now you\nlaw that you all know about and now you divide<01:14:51.840><c> by</c><01:14:52.639><c> um</c><01:14:53.320><c> the</c><01:14:54.239><c> the</c><01:14:54.400><c> exponential</c>\ndivide by um the the exponential\ndivide by um the the exponential reward<01:14:56.520><c> uh</c><01:14:56.800><c> on</c><01:14:57.040><c> the</c><01:14:57.360><c> first</c><01:14:58.080><c> example</c><01:14:58.880><c> sorry</c><01:14:59.120><c> on</c>\nreward uh on the first example sorry on\nreward uh on the first example sorry on the<01:14:59.320><c> first</c><01:14:59.520><c> output</c><01:15:00.080><c> and</c><01:15:00.199><c> this</c><01:15:00.320><c> is</c><01:15:00.400><c> on</c><01:15:00.520><c> the</c>\nthe first output and this is on the\nthe first output and this is on the second<01:15:00.880><c> output</c><01:15:01.400><c> and</c><01:15:01.480><c> you</c><01:15:01.639><c> basically</c><01:15:02.000><c> train</c><01:15:02.679><c> so</c>\nsecond output and you basically train so\nsecond output and you basically train so the<01:15:03.080><c> reason</c><01:15:03.320><c> why</c><01:15:03.440><c> you</c><01:15:03.560><c> do</c><01:15:03.719><c> that</c><01:15:03.920><c> is</c><01:15:04.000><c> that</c><01:15:04.120><c> you</c>\nthe reason why you do that is that you\nthe reason why you do that is that you train<01:15:04.719><c> your</c><01:15:05.040><c> your</c><01:15:05.199><c> model</c><01:15:05.760><c> you</c><01:15:05.920><c> train</c><01:15:06.199><c> this</c>\ntrain your your model you train this\ntrain your your model you train this reward<01:15:06.719><c> model</c><01:15:07.199><c> to</c><01:15:07.360><c> be</c><01:15:07.520><c> able</c><01:15:07.719><c> to</c><01:15:08.320><c> classify</c><01:15:09.320><c> um</c>\nreward model to be able to classify um\nreward model to be able to classify um how<01:15:10.679><c> much</c><01:15:11.159><c> better</c><01:15:11.679><c> one</c><01:15:11.960><c> output</c><01:15:12.360><c> is</c><01:15:12.520><c> to</c><01:15:12.719><c> another</c>\nhow much better one output is to another\nhow much better one output is to another one<01:15:13.440><c> so</c><01:15:13.679><c> another</c><01:15:14.520><c> uh</c><01:15:14.639><c> slightly</c><01:15:15.000><c> less</c>\none so another uh slightly less\none so another uh slightly less convoluted<01:15:15.719><c> way</c><01:15:15.840><c> of</c><01:15:15.960><c> saying</c><01:15:16.239><c> it</c><01:15:16.520><c> is</c><01:15:16.639><c> that</c><01:15:16.760><c> your</c>\nconvoluted way of saying it is that your\nconvoluted way of saying it is that your reward<01:15:17.320><c> model</c><01:15:18.040><c> will</c><01:15:18.639><c> output</c><01:15:19.080><c> some</c><01:15:19.239><c> reward</c>\nreward model will output some reward\nreward model will output some reward that<01:15:19.679><c> will</c><01:15:19.840><c> be</c><01:15:19.960><c> used</c><01:15:20.239><c> as</c><01:15:20.320><c> the</c><01:15:20.480><c> logits</c><01:15:21.400><c> of</c><01:15:21.560><c> your</c>\nthat will be used as the logits of your\nthat will be used as the logits of your soft<01:15:22.120><c> Max</c><01:15:22.719><c> so</c><01:15:22.960><c> now</c><01:15:23.199><c> if</c><01:15:23.320><c> you</c><01:15:23.520><c> have</c><01:15:24.040><c> high</c><01:15:24.520><c> logic</c>\nsoft Max so now if you have high logic\nsoft Max so now if you have high logic in<01:15:25.159><c> your</c><01:15:25.280><c> softmax</c><01:15:26.000><c> it</c><01:15:26.080><c> means</c><01:15:26.360><c> that</c><01:15:26.520><c> you</c><01:15:27.239><c> highly</c>\nin your softmax it means that you highly\nin your softmax it means that you highly likely<01:15:28.520><c> this</c><01:15:29.360><c> um</c><01:15:29.920><c> output</c><01:15:30.480><c> is</c>\nlikely this um output is\nlikely this um output is better<01:15:32.679><c> uh</c><01:15:32.800><c> so</c><01:15:32.960><c> that's</c><01:15:33.120><c> what</c><01:15:33.199><c> we</c><01:15:33.320><c> call</c><01:15:33.560><c> Bradley</c>\nbetter uh so that's what we call Bradley\nbetter uh so that's what we call Bradley ter<01:15:34.239><c> model</c><01:15:35.159><c> yes</c><01:15:35.480><c> is</c><01:15:35.600><c> this</c><01:15:35.760><c> reward</c><01:15:36.080><c> model</c><01:15:36.520><c> going</c>\nter model yes is this reward model going\nter model yes is this reward model going over<01:15:37.159><c> the</c><01:15:37.480><c> entire</c><01:15:38.080><c> output</c><01:15:38.520><c> or</c><01:15:38.679><c> is</c><01:15:38.760><c> it</c>\nover the entire output or is it\nover the entire output or is it going<01:15:40.480><c> um</c><01:15:41.080><c> so</c><01:15:42.080><c> this</c><01:15:42.320><c> takes</c><01:15:42.880><c> the</c>\ngoing um so this takes the\ngoing um so this takes the entire<01:15:44.920><c> uh</c><01:15:45.440><c> yeah</c><01:15:45.560><c> this</c><01:15:45.679><c> takes</c><01:15:45.880><c> the</c><01:15:46.000><c> entire</c>\nentire uh yeah this takes the entire\nentire uh yeah this takes the entire output<01:15:46.600><c> at</c><01:15:46.719><c> once</c><01:15:46.880><c> so</c><01:15:47.000><c> it</c><01:15:47.080><c> takes</c><01:15:47.320><c> all</c><01:15:47.520><c> the</c><01:15:47.600><c> input</c>\noutput at once so it takes all the input\noutput at once so it takes all the input and<01:15:48.000><c> all</c><01:15:48.159><c> the</c><01:15:48.239><c> output</c><01:15:48.679><c> and</c><01:15:48.800><c> it</c><01:15:48.960><c> gives</c><01:15:49.199><c> one</c>\nand all the output and it gives one\nand all the output and it gives one number\nyes<01:15:53.880><c> would</c><01:15:54.080><c> human</c><01:15:54.440><c> be</c><01:15:55.199><c> sorry</c><01:15:55.760><c> with</c><01:15:55.880><c> the</c><01:15:56.080><c> reward</c>\nyes would human be sorry with the reward\nyes would human be sorry with the reward model<01:15:57.040><c> where</c><01:15:57.159><c> would</c><01:15:57.280><c> a</c><01:15:57.480><c> human</c><01:15:57.840><c> be</c><01:15:58.400><c> like</c><01:15:58.760><c> oh</c><01:15:58.920><c> I</c>\nmodel where would a human be like oh I\nmodel where would a human be like oh I see<01:16:00.440><c> okay</c><01:16:00.600><c> sorry</c><01:16:01.199><c> maybe</c><01:16:01.360><c> I</c><01:16:01.440><c> wasn't</c><01:16:01.719><c> clear</c><01:16:02.520><c> um</c>\nsee okay sorry maybe I wasn't clear um\nsee okay sorry maybe I wasn't clear um you<01:16:03.400><c> train</c><01:16:03.800><c> this</c><01:16:03.960><c> reward</c><01:16:04.400><c> model</c><01:16:05.199><c> to</c><01:16:05.639><c> fit</c><01:16:06.440><c> this</c>\nyou train this reward model to fit this\nyou train this reward model to fit this green<01:16:07.120><c> and</c><01:16:07.440><c> and</c><01:16:07.920><c> red</c><01:16:08.600><c> preference</c><01:16:09.040><c> from</c><01:16:09.280><c> humans</c>\ngreen and and red preference from humans\ngreen and and red preference from humans so<01:16:10.159><c> basically</c><01:16:10.560><c> you</c><01:16:10.679><c> train</c><01:16:10.960><c> a</c><01:16:11.159><c> classifier</c><01:16:12.040><c> to</c>\nso basically you train a classifier to\nso basically you train a classifier to say<01:16:12.600><c> whether</c><01:16:13.199><c> the</c><01:16:13.360><c> humans</c><01:16:13.719><c> prefer</c><01:16:14.080><c> red</c><01:16:14.239><c> or</c>\nsay whether the humans prefer red or\nsay whether the humans prefer red or green<01:16:15.920><c> uh</c><01:16:16.040><c> but</c><01:16:16.239><c> instead</c><01:16:16.520><c> of</c><01:16:16.639><c> using</c><01:16:16.960><c> the</c><01:16:17.199><c> binary</c>\ngreen uh but instead of using the binary\ngreen uh but instead of using the binary reward<01:16:18.199><c> which</c><01:16:18.320><c> is</c><01:16:18.440><c> what</c><01:16:18.560><c> the</c><01:16:18.679><c> human</c><01:16:19.000><c> would</c>\nreward which is what the human would\nreward which is what the human would tell<01:16:19.360><c> you</c><01:16:19.880><c> you</c><01:16:20.120><c> basically</c><01:16:20.480><c> use</c><01:16:20.679><c> the</c><01:16:20.880><c> logits</c><01:16:21.880><c> of</c>\ntell you you basically use the logits of\ntell you you basically use the logits of the<01:16:22.199><c> soft</c><01:16:22.560><c> Max</c><01:16:23.320><c> and</c><01:16:23.440><c> the</c><01:16:23.560><c> thing</c><01:16:23.719><c> with</c><01:16:23.840><c> the</c>\nthe soft Max and the thing with the\nthe soft Max and the thing with the logits<01:16:24.639><c> is</c><01:16:24.760><c> that</c><01:16:24.920><c> that</c><01:16:25.040><c> logits</c><01:16:25.520><c> are</c>\nlogits is that that logits are\nlogits is that that logits are continuous<01:16:26.719><c> so</c><01:16:26.960><c> now</c><01:16:27.120><c> you</c><01:16:27.239><c> know</c><01:16:27.440><c> that</c><01:16:27.600><c> if</c><01:16:27.679><c> your</c>\ncontinuous so now you know that if your\ncontinuous so now you know that if your reward<01:16:28.199><c> model</c><01:16:28.600><c> said</c><01:16:29.159><c> it</c><01:16:29.360><c> has</c><01:16:29.600><c> high</c><01:16:29.760><c> logits</c>\nreward model said it has high logits\nreward model said it has high logits then<01:16:30.880><c> in</c><01:16:31.040><c> some</c><01:16:31.280><c> ways</c><01:16:31.560><c> the</c><01:16:31.760><c> human</c><01:16:32.480><c> highly</c>\nthen in some ways the human highly\nthen in some ways the human highly prefer<01:16:33.280><c> this</c><01:16:33.480><c> answer</c><01:16:33.920><c> to</c><01:16:34.120><c> some</c><01:16:34.360><c> other</c>\nprefer this answer to some other\nprefer this answer to some other answer<01:16:37.199><c> great</c><01:16:38.040><c> um</c><01:16:38.960><c> so</c><01:16:39.159><c> as</c><01:16:39.239><c> I</c><01:16:39.400><c> just</c><01:16:39.560><c> said</c>\nanswer great um so as I just said\nanswer great um so as I just said continuous<01:16:40.440><c> information</c><01:16:40.880><c> so</c><01:16:41.040><c> it's</c><01:16:41.199><c> better</c><01:16:41.440><c> so</c>\ncontinuous information so it's better so\ncontinuous information so it's better so that's<01:16:41.760><c> what</c><01:16:41.880><c> people</c><01:16:42.600><c> uh</c><01:16:42.719><c> use</c><01:16:42.960><c> in</c><01:16:43.159><c> practice</c><01:16:43.600><c> or</c>\nthat's what people uh use in practice or\nthat's what people uh use in practice or at<01:16:43.840><c> least</c><01:16:44.159><c> used</c><01:16:44.400><c> to</c><01:16:44.520><c> use</c><01:16:44.719><c> in</c><01:16:44.880><c> practice</c><01:16:45.560><c> I'll</c>\nat least used to use in practice I'll\nat least used to use in practice I'll tell<01:16:45.920><c> you</c><01:16:46.120><c> about</c><01:16:46.560><c> uh</c><01:16:46.719><c> the</c><01:16:46.840><c> other</c><01:16:47.040><c> algorithm</c>\ntell you about uh the other algorithm\ntell you about uh the other algorithm later<01:16:48.320><c> uh</c><01:16:48.400><c> so</c><01:16:48.560><c> what</c><01:16:48.639><c> you</c><01:16:48.719><c> do</c><01:16:48.960><c> at</c><01:16:49.040><c> the</c><01:16:49.159><c> end</c><01:16:49.600><c> is</c>\nlater uh so what you do at the end is\nlater uh so what you do at the end is that<01:16:49.880><c> you</c><01:16:50.080><c> basically</c><01:16:50.560><c> try</c><01:16:50.840><c> to</c><01:16:51.600><c> just</c><01:16:51.760><c> use</c>\nthat you basically try to just use\nthat you basically try to just use reinforcement<01:16:52.480><c> learning</c><01:16:52.800><c> that</c><01:16:52.880><c> you</c><01:16:53.000><c> know</c>\nreinforcement learning that you know\nreinforcement learning that you know about<01:16:53.639><c> now</c><01:16:53.800><c> we</c><01:16:53.920><c> know</c><01:16:54.480><c> we</c><01:16:54.639><c> have</c><01:16:55.120><c> reward</c><01:16:55.760><c> what</c>\nabout now we know we have reward what\nabout now we know we have reward what you<01:16:56.080><c> sample</c><01:16:56.480><c> through</c><01:16:57.080><c> is</c><01:16:57.280><c> the</c><01:16:57.600><c> generation</c>\nyou sample through is the generation\nyou sample through is the generation from<01:16:58.239><c> your</c><01:16:58.360><c> large</c><01:16:58.600><c> language</c><01:16:58.920><c> model</c><01:16:59.920><c> um</c><01:17:00.239><c> and</c>\nfrom your large language model um and\nfrom your large language model um and then<01:17:00.560><c> you</c><01:17:00.679><c> just</c><01:17:00.800><c> use</c><01:17:01.040><c> some</c><01:17:01.199><c> regularization</c>\nthen you just use some regularization\nthen you just use some regularization term<01:17:02.120><c> so</c><01:17:02.239><c> the</c><01:17:02.360><c> reason</c><01:17:02.600><c> why</c><01:17:02.719><c> you</c><01:17:03.000><c> do</c><01:17:03.159><c> this</c>\nterm so the reason why you do this\nterm so the reason why you do this regularization<01:17:03.960><c> term</c><01:17:04.360><c> is</c><01:17:04.560><c> for</c><01:17:04.920><c> avoiding</c><01:17:05.280><c> what</c>\nregularization term is for avoiding what\nregularization term is for avoiding what we<01:17:05.520><c> call</c><01:17:05.679><c> over</c><01:17:05.960><c> optimization</c><01:17:06.719><c> so</c><01:17:06.920><c> this</c><01:17:07.120><c> reward</c>\nwe call over optimization so this reward\nwe call over optimization so this reward model<01:17:07.800><c> might</c><01:17:07.960><c> not</c><01:17:08.159><c> be</c><01:17:08.360><c> really</c><01:17:08.639><c> represent</c><01:17:09.159><c> like</c>\nmodel might not be really represent like\nmodel might not be really represent like might<01:17:09.679><c> not</c><01:17:10.000><c> perfectly</c><01:17:10.400><c> model</c><01:17:11.040><c> human</c>\nmight not perfectly model human\nmight not perfectly model human preferences<01:17:12.199><c> so</c><01:17:12.320><c> you</c><01:17:12.440><c> don't</c><01:17:12.639><c> want</c><01:17:12.719><c> to</c>\npreferences so you don't want to\npreferences so you don't want to maximize<01:17:13.560><c> this</c><01:17:13.679><c> thing</c><01:17:14.159><c> to</c><01:17:14.600><c> essentially</c>\nmaximize this thing to essentially\nmaximize this thing to essentially Infinity<01:17:16.360><c> um</c><01:17:17.360><c> and</c><01:17:17.520><c> you</c><01:17:17.679><c> do</c><01:17:18.080><c> it</c><01:17:18.280><c> using</c><01:17:18.800><c> uh</c><01:17:19.000><c> po</c>\nInfinity um and you do it using uh po\nInfinity um and you do it using uh po which<01:17:19.960><c> is</c><01:17:20.159><c> a</c><01:17:21.159><c> common</c><01:17:21.960><c> uh</c><01:17:22.239><c> reinforcement</c>\nwhich is a common uh reinforcement\nwhich is a common uh reinforcement learning<01:17:23.239><c> algorithm</c><01:17:24.239><c> um</c><01:17:24.560><c> one</c><01:17:24.960><c> thing</c><01:17:25.080><c> to</c><01:17:25.239><c> note</c>\nlearning algorithm um one thing to note\nlearning algorithm um one thing to note here<01:17:25.800><c> because</c><01:17:25.920><c> it</c><01:17:26.000><c> will</c><01:17:26.159><c> be</c><01:17:26.280><c> important</c><01:17:26.560><c> for</c>\nhere because it will be important for\nhere because it will be important for later<01:17:27.560><c> is</c><01:17:27.760><c> that</c><01:17:27.960><c> when</c><01:17:28.159><c> we</c><01:17:28.679><c> use</c><01:17:29.080><c> maximum</c>\nlater is that when we use maximum\nlater is that when we use maximum likelihood\nlikelihood\nlikelihood um<01:17:32.800><c> sorry</c><01:17:33.600><c> now</c><01:17:33.960><c> the</c><01:17:34.159><c> large</c><01:17:34.440><c> language</c><01:17:34.719><c> models</c>\num sorry now the large language models\num sorry now the large language models are<01:17:35.159><c> actually</c><01:17:35.360><c> a</c><01:17:35.639><c> policy</c><01:17:36.639><c> for</c><01:17:36.840><c> your</c>\nare actually a policy for your\nare actually a policy for your reinforcement<01:17:37.600><c> learning</c><01:17:38.239><c> it's</c><01:17:38.520><c> not</c>\nreinforcement learning it's not\nreinforcement learning it's not maximizing<01:17:39.639><c> maximum</c><01:17:40.080><c> likelihood</c><01:17:40.520><c> anymore</c>\nmaximizing maximum likelihood anymore\nmaximizing maximum likelihood anymore which<01:17:41.360><c> means</c><01:17:41.600><c> that</c><01:17:41.760><c> you're</c><01:17:41.920><c> not</c><01:17:42.159><c> modeling</c><01:17:42.600><c> any</c>\nwhich means that you're not modeling any\nwhich means that you're not modeling any distribution<01:17:43.400><c> anymore</c><01:17:44.199><c> and</c><01:17:44.320><c> the</c><01:17:44.440><c> reason</c><01:17:44.719><c> why</c>\ndistribution anymore and the reason why\ndistribution anymore and the reason why this<01:17:44.960><c> is</c><01:17:45.120><c> important</c><01:17:45.440><c> is</c><01:17:45.560><c> that</c><01:17:45.840><c> models</c><01:17:46.239><c> that</c>\nthis is important is that models that\nthis is important is that models that went<01:17:46.679><c> through</c><01:17:47.280><c> this</c><01:17:47.480><c> type</c><01:17:47.639><c> of</c><01:17:47.920><c> Po</c><01:17:48.920><c> actually</c>\nwent through this type of Po actually\nwent through this type of Po actually don't<01:17:49.560><c> give</c><01:17:49.760><c> you</c><01:17:50.400><c> likelihoods</c><01:17:50.960><c> of</c><01:17:51.199><c> text</c><01:17:51.719><c> that</c>\ndon't give you likelihoods of text that\ndon't give you likelihoods of text that are<01:17:52.000><c> meaningful</c><01:17:52.760><c> cuz</c><01:17:53.040><c> what</c><01:17:53.159><c> you</c><01:17:53.400><c> optimize</c>\nare meaningful cuz what you optimize\nare meaningful cuz what you optimize them<01:17:54.239><c> to</c><01:17:54.400><c> do</c><01:17:54.560><c> is</c><01:17:54.679><c> B</c><01:17:54.840><c> basically</c><01:17:55.199><c> just</c><01:17:55.440><c> optimized</c>\nthem to do is B basically just optimized\nthem to do is B basically just optimized for<01:17:56.239><c> generating</c><01:17:56.760><c> the</c><01:17:56.920><c> most</c><01:17:57.199><c> likely</c><01:17:57.560><c> thing</c><01:17:58.480><c> not</c>\nfor generating the most likely thing not\nfor generating the most likely thing not optimize<01:17:59.360><c> for</c><01:17:59.560><c> modeling</c><01:18:00.080><c> like</c><01:18:00.560><c> all</c><01:18:00.719><c> the</c>\noptimize for modeling like all the\noptimize for modeling like all the answers<01:18:01.239><c> that</c><01:18:01.400><c> humans</c><01:18:01.760><c> might</c><01:18:01.960><c> say</c><01:18:02.440><c> another</c>\nanswers that humans might say another\nanswers that humans might say another way<01:18:02.920><c> of</c><01:18:03.040><c> saying</c><01:18:03.320><c> that</c><01:18:03.679><c> is</c><01:18:03.840><c> that</c><01:18:04.000><c> there's</c>\nway of saying that is that there's\nway of saying that is that there's nothing<01:18:04.679><c> that</c><01:18:04.880><c> incentivizes</c><01:18:05.760><c> here</c><01:18:06.080><c> the</c><01:18:06.280><c> model</c>\nnothing that incentivizes here the model\nnothing that incentivizes here the model to<01:18:07.159><c> not</c><01:18:07.520><c> give</c><01:18:08.120><c> a</c><01:18:08.480><c> like</c><01:18:08.719><c> a</c><01:18:09.239><c> um</c><01:18:09.719><c> a</c><01:18:09.880><c> single</c>\nto not give a like a um a single\nto not give a like a um a single possible<01:18:10.719><c> generation</c><01:18:11.600><c> nothing</c><01:18:12.000><c> here</c><01:18:12.719><c> says</c>\npossible generation nothing here says\npossible generation nothing here says it's<01:18:13.400><c> good</c><01:18:13.920><c> if</c><01:18:14.040><c> you</c><01:18:14.239><c> have</c><01:18:14.520><c> some</c><01:18:14.760><c> distribution</c>\nit's good if you have some distribution\nit's good if you have some distribution with<01:18:15.440><c> some</c>\nwith some\nwith some entropy<01:18:17.199><c> um</c><01:18:17.880><c> okay</c><01:18:18.080><c> if</c><01:18:18.120><c> you</c><01:18:18.239><c> haven't</c><01:18:18.400><c> followed</c>\nentropy um okay if you haven't followed\nentropy um okay if you haven't followed it's<01:18:18.840><c> not</c><01:18:19.040><c> that</c><01:18:19.199><c> important</c><01:18:19.639><c> but</c><01:18:20.040><c> just</c><01:18:20.360><c> good</c><01:18:20.480><c> to</c>\nit's not that important but just good to\nit's not that important but just good to knowe<01:18:22.440><c> great</c><01:18:23.440><c> so</c><01:18:23.840><c> PO</c><01:18:24.320><c> is</c><01:18:24.480><c> exact</c><01:18:25.080><c> what</c><01:18:25.400><c> chat</c><01:18:25.679><c> GPT</c>\nknowe great so PO is exact what chat GPT\nknowe great so PO is exact what chat GPT did<01:18:26.560><c> originally</c><01:18:27.320><c> so</c><01:18:27.600><c> here's</c><01:18:27.960><c> the</c><01:18:28.199><c> on</c><01:18:28.400><c> the</c><01:18:28.560><c> blog</c>\ndid originally so here's the on the blog\ndid originally so here's the on the blog post<01:18:29.360><c> or</c><01:18:30.159><c> what</c><01:18:30.320><c> they</c><01:18:30.520><c> have</c><01:18:30.920><c> is</c><01:18:31.159><c> step</c><01:18:31.440><c> one</c><01:18:31.880><c> do</c>\npost or what they have is step one do\npost or what they have is step one do supervise<01:18:32.639><c> fine</c><01:18:32.840><c> training</c><01:18:33.360><c> which</c><01:18:33.520><c> now</c><01:18:33.679><c> you</c>\nsupervise fine training which now you\nsupervise fine training which now you all<01:18:33.960><c> know</c><01:18:34.159><c> about</c><01:18:34.800><c> step</c><01:18:35.040><c> two</c><01:18:35.480><c> train</c><01:18:35.760><c> a</c><01:18:35.920><c> reward</c>\nall know about step two train a reward\nall know about step two train a reward model<01:18:36.600><c> on</c><01:18:36.800><c> human</c><01:18:37.080><c> preferences</c><01:18:38.040><c> step</c><01:18:38.320><c> three</c><01:18:38.840><c> do</c>\nmodel on human preferences step three do\nmodel on human preferences step three do po<01:18:39.960><c> multiple</c><01:18:40.400><c> steps</c><01:18:40.880><c> which</c><01:18:41.040><c> is</c><01:18:41.199><c> where</c><01:18:41.360><c> you</c><01:18:41.480><c> see</c>\npo multiple steps which is where you see\npo multiple steps which is where you see this<01:18:42.280><c> this</c><01:18:42.440><c> blue</c><01:18:42.719><c> arrow</c><01:18:43.120><c> so</c><01:18:43.280><c> you</c><01:18:43.400><c> continue</c><01:18:43.880><c> you</c>\nthis this blue arrow so you continue you\nthis this blue arrow so you continue you train<01:18:44.199><c> the</c><01:18:44.320><c> model</c><01:18:44.560><c> once</c><01:18:44.719><c> with</c><01:18:44.960><c> po</c><01:18:45.520><c> you</c><01:18:45.639><c> collect</c>\ntrain the model once with po you collect\ntrain the model once with po you collect new<01:18:46.120><c> data</c><01:18:46.360><c> you</c><01:18:46.480><c> continue</c><01:18:47.480><c> uh</c><01:18:47.639><c> and</c><01:18:47.800><c> that's</c><01:18:48.040><c> why</c>\nnew data you continue uh and that's why\nnew data you continue uh and that's why and<01:18:48.440><c> that's</c><01:18:48.639><c> exactly</c><01:18:48.960><c> what</c><01:18:49.080><c> Chad</c><01:18:49.320><c> GPT</c><01:18:49.719><c> did</c><01:18:50.480><c> uh</c>\nand that's exactly what Chad GPT did uh\nand that's exactly what Chad GPT did uh that<01:18:50.760><c> was</c><01:18:50.880><c> a</c><01:18:51.120><c> big</c><01:18:51.320><c> breakthrough</c><01:18:52.199><c> between</c><01:18:52.800><c> gpt3</c>\nthat was a big breakthrough between gpt3\nthat was a big breakthrough between gpt3 and<01:18:53.679><c> Chad</c><01:18:53.920><c> GPT</c>\nand Chad GPT\nand Chad GPT one<01:18:55.560><c> thing</c><01:18:55.679><c> to</c><01:18:55.880><c> note</c><01:18:56.440><c> is</c><01:18:56.719><c> that</c><01:18:57.199><c> uh</c><01:18:57.360><c> P</c><01:18:57.800><c> has</c><01:18:57.920><c> many</c>\none thing to note is that uh P has many\none thing to note is that uh P has many challenges<01:18:59.040><c> reinforcement</c><01:18:59.679><c> learning</c><01:18:59.960><c> is</c>\nchallenges reinforcement learning is\nchallenges reinforcement learning is something<01:19:00.360><c> that's</c><01:19:00.719><c> super</c><01:19:01.159><c> nice</c>\nsomething that's super nice\nsomething that's super nice theoretically<01:19:02.320><c> in</c><01:19:02.520><c> practice</c><01:19:02.960><c> anyone</c><01:19:03.239><c> who</c>\ntheoretically in practice anyone who\ntheoretically in practice anyone who ever<01:19:03.600><c> worked</c><01:19:03.800><c> with</c><01:19:03.920><c> reinforcement</c><01:19:04.440><c> learning</c>\never worked with reinforcement learning\never worked with reinforcement learning knows<01:19:05.000><c> it's</c><01:19:05.320><c> such</c><01:19:05.520><c> a</c><01:19:05.679><c> mess</c><01:19:06.400><c> uh</c><01:19:06.520><c> there's</c><01:19:06.719><c> a</c><01:19:06.880><c> lot</c>\nknows it's such a mess uh there's a lot\nknows it's such a mess uh there's a lot of<01:19:07.199><c> things</c><01:19:07.480><c> like</c><01:19:07.719><c> roll</c><01:19:07.960><c> outs</c><01:19:08.280><c> out</c><01:19:08.480><c> of</c><01:19:08.639><c> Loops</c>\nof things like roll outs out of Loops\nof things like roll outs out of Loops clipping<01:19:09.600><c> so</c><01:19:09.800><c> many</c><01:19:10.440><c> complications</c><01:19:11.440><c> um</c><01:19:12.120><c> so</c>\nclipping so many complications um so\nclipping so many complications um so it's<01:19:12.440><c> messy</c><01:19:13.159><c> this</c><01:19:13.280><c> is</c><01:19:13.480><c> the</c><01:19:13.679><c> idealized</c><01:19:14.280><c> PO</c><01:19:14.679><c> used</c>\nit's messy this is the idealized PO used\nit's messy this is the idealized PO used for<01:19:15.080><c> LM</c><01:19:15.480><c> settings</c><01:19:15.840><c> so</c><01:19:16.040><c> that's</c><01:19:16.239><c> already</c><01:19:16.560><c> much</c>\nfor LM settings so that's already much\nfor LM settings so that's already much more<01:19:16.920><c> complicated</c><01:19:17.440><c> than</c><01:19:17.560><c> this</c><01:19:17.800><c> expectation</c>\nmore complicated than this expectation\nmore complicated than this expectation we<01:19:18.320><c> saw</c><01:19:18.560><c> before</c><01:19:19.199><c> and</c><01:19:19.320><c> in</c><01:19:19.520><c> practice</c><01:19:19.800><c> it's</c>\nwe saw before and in practice it's\nwe saw before and in practice it's actually<01:19:20.199><c> much</c><01:19:20.360><c> more</c><01:19:20.560><c> complicated</c><01:19:21.000><c> so</c><01:19:21.120><c> we</c>\nactually much more complicated so we\nactually much more complicated so we have<01:19:21.440><c> one</c><01:19:21.639><c> implementation</c><01:19:22.239><c> of</c><01:19:22.360><c> it</c><01:19:22.520><c> that</c><01:19:22.600><c> we</c>\nhave one implementation of it that we\nhave one implementation of it that we had<01:19:22.880><c> to</c><01:19:23.000><c> do</c><01:19:23.639><c> and</c><01:19:23.760><c> I'm</c><01:19:24.080><c> not</c><01:19:24.239><c> going</c><01:19:24.320><c> to</c><01:19:24.480><c> go</c>\nhad to do and I'm not going to go\nhad to do and I'm not going to go through<01:19:24.920><c> it</c><01:19:25.080><c> but</c><01:19:25.199><c> basically</c><01:19:25.560><c> you</c><01:19:25.679><c> have</c><01:19:25.880><c> like</c>\nthrough it but basically you have like\nthrough it but basically you have like so<01:19:26.360><c> much</c><01:19:26.639><c> stuff</c><01:19:26.880><c> that</c><01:19:27.000><c> you</c><01:19:27.159><c> have</c><01:19:27.320><c> to</c><01:19:27.560><c> think</c>\nso much stuff that you have to think\nso much stuff that you have to think about<01:19:28.280><c> when</c><01:19:28.440><c> you</c><01:19:28.600><c> implement</c><01:19:29.159><c> that</c><01:19:29.360><c> type</c><01:19:29.639><c> of</c><01:19:30.440><c> of</c>\nabout when you implement that type of of\nabout when you implement that type of of uh<01:19:30.960><c> po</c><01:19:31.360><c> algorithm</c><01:19:31.760><c> so</c><01:19:31.880><c> you</c><01:19:31.960><c> have</c><01:19:32.080><c> clipping</c>\nuh po algorithm so you have clipping\nuh po algorithm so you have clipping everywhere<01:19:33.120><c> you</c><01:19:33.280><c> have</c><01:19:33.560><c> a</c><01:19:33.679><c> lot</c><01:19:33.800><c> of</c>\neverywhere you have a lot of\neverywhere you have a lot of complexities<01:19:35.080><c> and</c><01:19:35.440><c> things</c><01:19:35.600><c> are</c><01:19:35.719><c> not</c><01:19:35.880><c> well</c>\ncomplexities and things are not well\ncomplexities and things are not well documented<01:19:37.600><c> all</c><01:19:37.840><c> this</c><01:19:38.000><c> to</c><01:19:38.239><c> say</c><01:19:39.120><c> um</c><01:19:39.560><c> that</c><01:19:39.880><c> we're</c>\ndocumented all this to say um that we're\ndocumented all this to say um that we're going<01:19:40.159><c> to</c><01:19:40.480><c> there</c><01:19:40.679><c> was</c><01:19:40.800><c> a</c><01:19:40.960><c> new</c><01:19:41.320><c> method</c><01:19:41.679><c> that</c><01:19:41.800><c> was</c>\ngoing to there was a new method that was\ngoing to there was a new method that was proposed<01:19:42.639><c> uh</c><01:19:42.760><c> also</c><01:19:43.280><c> from</c><01:19:43.440><c> Sanford</c><01:19:44.000><c> one</c><01:19:44.199><c> year</c>\nproposed uh also from Sanford one year\nproposed uh also from Sanford one year ago<01:19:44.760><c> called</c><01:19:45.000><c> DPO</c><01:19:45.840><c> which</c><01:19:45.960><c> is</c><01:19:46.120><c> essentially</c><01:19:46.440><c> a</c>\nago called DPO which is essentially a\nago called DPO which is essentially a simplification<01:19:47.600><c> of</c><01:19:48.000><c> Po</c><01:19:49.000><c> um</c><01:19:49.880><c> and</c><01:19:50.040><c> the</c><01:19:50.280><c> way</c><01:19:51.280><c> uh</c>\nsimplification of Po um and the way uh\nsimplification of Po um and the way uh what<01:19:51.840><c> they</c><01:19:52.000><c> did</c><01:19:52.360><c> or</c><01:19:52.520><c> the</c><01:19:52.639><c> idea</c><01:19:52.920><c> that</c><01:19:53.080><c> they</c><01:19:53.239><c> have</c>\nwhat they did or the idea that they have\nwhat they did or the idea that they have is<01:19:53.880><c> that</c><01:19:54.080><c> instead</c><01:19:54.320><c> of</c><01:19:54.440><c> using</c><01:19:54.840><c> reinforcement</c>\nis that instead of using reinforcement\nis that instead of using reinforcement learning<01:19:56.280><c> you</c><01:19:56.360><c> can</c><01:19:56.520><c> just</c><01:19:56.719><c> maximize</c><01:19:57.199><c> the</c>\nlearning you can just maximize the\nlearning you can just maximize the probability<01:19:57.719><c> of</c><01:19:57.840><c> generating</c><01:19:58.360><c> the</c><01:19:58.560><c> stuff</c><01:19:58.840><c> that</c>\nprobability of generating the stuff that\nprobability of generating the stuff that you<01:19:59.120><c> like</c><01:19:59.600><c> and</c><01:19:59.760><c> minimizing</c><01:20:00.239><c> the</c><01:20:00.360><c> probability</c>\nyou like and minimizing the probability\nyou like and minimizing the probability of<01:20:00.760><c> the</c><01:20:00.880><c> stuff</c><01:20:01.040><c> that</c><01:20:01.120><c> you</c><01:20:01.239><c> don't</c><01:20:01.520><c> like</c><01:20:02.320><c> uh</c><01:20:02.480><c> so</c>\nof the stuff that you don't like uh so\nof the stuff that you don't like uh so if<01:20:03.000><c> you</c><01:20:03.120><c> think</c><01:20:03.280><c> about</c><01:20:03.440><c> the</c><01:20:03.520><c> human</c><01:20:03.760><c> preference</c>\nif you think about the human preference\nif you think about the human preference the<01:20:04.360><c> red</c><01:20:04.520><c> and</c><01:20:04.719><c> green</c><01:20:05.440><c> maximize</c><01:20:06.440><c> uh</c><01:20:06.560><c> green</c>\nthe red and green maximize uh green\nthe red and green maximize uh green minimize<01:20:07.600><c> red</c><01:20:08.600><c> um</c><01:20:09.000><c> so</c><01:20:09.199><c> the</c><01:20:09.360><c> loss</c><01:20:09.920><c> is</c><01:20:10.159><c> actually</c>\nminimize red um so the loss is actually\nminimize red um so the loss is actually this<01:20:10.880><c> one</c><01:20:11.639><c> uh</c><01:20:11.800><c> where</c><01:20:12.000><c> what</c><01:20:12.080><c> you</c><01:20:12.199><c> see</c><01:20:12.560><c> this</c><01:20:12.679><c> is</c>\nthis one uh where what you see this is\nthis one uh where what you see this is simply<01:20:13.679><c> um</c><01:20:14.440><c> some</c><01:20:14.920><c> log</c><01:20:15.320><c> of</c><01:20:15.560><c> the</c><01:20:16.040><c> model</c><01:20:16.719><c> so</c><01:20:16.880><c> this</c>\nsimply um some log of the model so this\nsimply um some log of the model so this is<01:20:17.120><c> the</c><01:20:17.239><c> likelihood</c><01:20:17.639><c> of</c><01:20:17.760><c> a</c><01:20:17.880><c> model</c><01:20:18.159><c> generating</c>\nis the likelihood of a model generating\nis the likelihood of a model generating the<01:20:18.840><c> things</c><01:20:19.120><c> that</c><01:20:19.239><c> the</c><01:20:19.400><c> human</c><01:20:19.719><c> preferred</c>\nthe things that the human preferred\nthe things that the human preferred given<01:20:20.840><c> the</c><01:20:21.199><c> the</c><01:20:21.480><c> inputs</c><01:20:22.679><c> um</c><01:20:23.679><c> and</c><01:20:24.120><c> what</c><01:20:24.239><c> you</c><01:20:24.400><c> try</c>\ngiven the the inputs um and what you try\ngiven the the inputs um and what you try to<01:20:24.920><c> do</c><01:20:25.040><c> is</c><01:20:25.199><c> basically</c>\nto do is basically\nto do is basically maximize<01:20:27.239><c> uh</c><01:20:27.679><c> the</c><01:20:28.600><c> likelihood</c><01:20:29.320><c> of</c><01:20:29.480><c> generating</c>\nmaximize uh the likelihood of generating\nmaximize uh the likelihood of generating the<01:20:29.920><c> things</c><01:20:30.120><c> that</c><01:20:30.199><c> you</c><01:20:30.360><c> like</c><01:20:30.600><c> minimize</c><01:20:31.040><c> the</c>\nthe things that you like minimize the\nthe things that you like minimize the likelihood<01:20:31.520><c> of</c><01:20:31.639><c> the</c><01:20:31.719><c> things</c><01:20:31.920><c> that</c><01:20:32.040><c> you</c><01:20:32.159><c> don't</c>\nlikelihood of the things that you don't\nlikelihood of the things that you don't like<01:20:33.199><c> um</c><01:20:34.080><c> all</c><01:20:34.280><c> the</c><01:20:34.440><c> rest</c><01:20:34.679><c> of</c><01:20:34.800><c> the</c><01:20:34.960><c> terms</c><01:20:35.560><c> here</c>\nlike um all the rest of the terms here\nlike um all the rest of the terms here it's<01:20:35.880><c> not</c><01:20:36.080><c> too</c><01:20:36.280><c> important</c><01:20:36.600><c> it's</c><01:20:36.800><c> actually</c>\nit's not too important it's actually\nit's not too important it's actually really<01:20:38.159><c> not</c><01:20:38.400><c> that</c><01:20:38.639><c> complicated</c><01:20:39.080><c> to</c>\nreally not that complicated to\nreally not that complicated to understand<01:20:39.920><c> but</c><01:20:40.040><c> at</c><01:20:40.159><c> a</c><01:20:40.280><c> high</c><01:20:40.440><c> level</c><01:20:40.920><c> it's</c>\nunderstand but at a high level it's\nunderstand but at a high level it's really<01:20:41.320><c> just</c><01:20:41.560><c> maximizing</c><01:20:42.400><c> the</c><01:20:42.520><c> things</c><01:20:42.679><c> you</c>\nreally just maximizing the things you\nreally just maximizing the things you like<01:20:43.120><c> minimizing</c><01:20:43.719><c> the</c><01:20:43.840><c> the</c><01:20:44.000><c> rest</c><01:20:44.800><c> um</c><01:20:45.800><c> and</c><01:20:46.639><c> one</c>\nlike minimizing the the rest um and one\nlike minimizing the the rest um and one thing<01:20:47.040><c> to</c><01:20:47.239><c> note</c><01:20:47.960><c> uh</c><01:20:48.120><c> which</c><01:20:48.239><c> I</c><01:20:48.320><c> was</c><01:20:48.440><c> going</c><01:20:48.560><c> to</c>\nthing to note uh which I was going to\nthing to note uh which I was going to say<01:20:48.840><c> just</c><01:20:49.280><c> here</c><01:20:49.679><c> is</c><01:20:49.800><c> that</c><01:20:50.000><c> actually</c><01:20:50.239><c> all</c><01:20:50.400><c> the</c>\nsay just here is that actually all the\nsay just here is that actually all the rest<01:20:50.760><c> is</c><01:20:50.920><c> chosen</c><01:20:51.520><c> such</c><01:20:51.920><c> that</c><01:20:52.760><c> um</c><01:20:53.080><c> the</c><01:20:53.239><c> global</c>\nrest is chosen such that um the global\nrest is chosen such that um the global Minima<01:20:54.440><c> of</c><01:20:54.679><c> of</c><01:20:55.040><c> Po</c><01:20:55.840><c> and</c><01:20:55.920><c> a</c><01:20:56.080><c> global</c><01:20:56.360><c> Minima</c><01:20:57.000><c> of</c>\nMinima of of Po and a global Minima of\nMinima of of Po and a global Minima of like<01:20:57.360><c> this</c><01:20:57.520><c> DPO</c><01:20:58.440><c> under</c><01:20:58.840><c> some</c><01:20:59.239><c> assumptions</c><01:20:59.760><c> are</c>\nlike this DPO under some assumptions are\nlike this DPO under some assumptions are essentially<01:21:00.360><c> equivalent</c><01:21:01.120><c> so</c><01:21:01.560><c> this</c><01:21:01.679><c> is</c><01:21:01.840><c> the</c>\nessentially equivalent so this is the\nessentially equivalent so this is the right<01:21:02.320><c> thing</c><01:21:02.719><c> to</c><01:21:02.880><c> do</c><01:21:03.600><c> mathematically</c><01:21:04.520><c> I'm</c><01:21:04.639><c> not</c>\nright thing to do mathematically I'm not\nright thing to do mathematically I'm not going<01:21:04.880><c> to</c><01:21:05.000><c> go</c><01:21:05.120><c> through</c><01:21:05.280><c> the</c><01:21:05.440><c> derivations</c><01:21:06.159><c> but</c>\ngoing to go through the derivations but\ngoing to go through the derivations but that's<01:21:06.920><c> the</c><01:21:07.040><c> right</c><01:21:07.199><c> thing</c><01:21:07.320><c> to</c><01:21:07.440><c> do</c><01:21:08.080><c> uh</c><01:21:08.199><c> it's</c>\nthat's the right thing to do uh it's\nthat's the right thing to do uh it's pretty<01:21:08.679><c> different</c><01:21:08.960><c> with</c><01:21:09.159><c> Po</c><01:21:09.520><c> in</c><01:21:09.600><c> the</c><01:21:09.719><c> sense</c>\npretty different with Po in the sense\npretty different with Po in the sense that<01:21:10.159><c> now</c><01:21:10.800><c> and</c><01:21:10.920><c> with</c><01:21:11.080><c> P</c><01:21:11.400><c> what</c><01:21:11.520><c> you</c><01:21:11.600><c> had</c><01:21:11.719><c> to</c><01:21:11.800><c> do</c>\nthat now and with P what you had to do\nthat now and with P what you had to do is<01:21:12.120><c> collect</c><01:21:12.400><c> the</c><01:21:12.520><c> human</c><01:21:12.760><c> preferences</c><01:21:13.560><c> then</c>\nis collect the human preferences then\nis collect the human preferences then train<01:21:13.920><c> a</c><01:21:14.280><c> uh</c><01:21:14.400><c> reward</c><01:21:14.719><c> model</c><01:21:15.040><c> with</c><01:21:15.159><c> maximum</c>\ntrain a uh reward model with maximum\ntrain a uh reward model with maximum likelihood<01:21:16.199><c> then</c><01:21:16.320><c> use</c><01:21:16.480><c> reinforcement</c>\nlikelihood then use reinforcement\nlikelihood then use reinforcement learning<01:21:17.560><c> now</c><01:21:17.760><c> all</c><01:21:17.880><c> you</c><01:21:18.000><c> do</c><01:21:18.199><c> is</c><01:21:18.320><c> basically</c>\nlearning now all you do is basically\nlearning now all you do is basically maximum<01:21:19.080><c> likelihood</c><01:21:19.800><c> much</c><01:21:20.000><c> simpler</c><01:21:20.679><c> yes</c><01:21:21.000><c> I</c>\nmaximum likelihood much simpler yes I\nmaximum likelihood much simpler yes I mean<01:21:21.320><c> yeah</c><01:21:21.480><c> so</c><01:21:21.600><c> it</c><01:21:21.719><c> seems</c><01:21:22.000><c> like</c><01:21:22.280><c> this</c><01:21:22.400><c> is</c><01:21:22.880><c> a</c>\nmean yeah so it seems like this is a\nmean yeah so it seems like this is a much<01:21:23.320><c> simpler</c><01:21:23.760><c> and</c><01:21:23.960><c> B</c><01:21:24.199><c> like</c><01:21:24.360><c> what</c><01:21:24.600><c> you</c><01:21:24.800><c> just</c>\nmuch simpler and B like what you just\nmuch simpler and B like what you just intuitively<01:21:25.880><c> do</c><01:21:26.159><c> if</c><01:21:26.719><c> this</c><01:21:27.320><c> why</c><01:21:27.520><c> did</c><01:21:28.000><c> they</c>\nintuitively do if this why did they\nintuitively do if this why did they start<01:21:28.600><c> with</c><01:21:28.840><c> this</c><01:21:29.000><c> reward</c><01:21:29.320><c> model</c><01:21:29.679><c> like</c><01:21:29.880><c> what</c>\nstart with this reward model like what\nstart with this reward model like what what<01:21:30.520><c> led</c><01:21:30.760><c> them</c><01:21:31.080><c> doing</c><01:21:31.440><c> that</c><01:21:31.800><c> I</c><01:21:31.880><c> think</c><01:21:32.000><c> it's</c><01:21:32.080><c> a</c>\nwhat led them doing that I think it's a\nwhat led them doing that I think it's a great<01:21:32.400><c> question</c><01:21:33.159><c> uh</c><01:21:33.280><c> I</c><01:21:33.360><c> don't</c><01:21:33.560><c> really</c><01:21:33.800><c> know</c>\ngreat question uh I don't really know\ngreat question uh I don't really know what<01:21:34.639><c> I</c><01:21:34.719><c> can</c><01:21:34.880><c> tell</c><01:21:35.080><c> you</c><01:21:35.320><c> is</c><01:21:35.480><c> that</c><01:21:35.880><c> at</c><01:21:36.120><c> open</c><01:21:36.520><c> ey</c>\nwhat I can tell you is that at open ey\nwhat I can tell you is that at open ey the<01:21:37.400><c> people</c><01:21:37.880><c> who</c><01:21:38.120><c> did</c><01:21:38.560><c> the</c><01:21:39.239><c> um</c><01:21:40.000><c> uh</c><01:21:40.120><c> who</c><01:21:40.320><c> did</c>\nthe people who did the um uh who did\nthe people who did the um uh who did basically<01:21:41.120><c> this</c><01:21:41.360><c> PP</c><01:21:41.719><c> uh</c><01:21:41.840><c> sorry</c><01:21:42.360><c> who</c><01:21:42.840><c> did</c><01:21:43.080><c> Chad</c>\nbasically this PP uh sorry who did Chad\nbasically this PP uh sorry who did Chad GPT<01:21:43.760><c> initially</c><01:21:44.600><c> are</c><01:21:44.840><c> the</c><01:21:45.000><c> ones</c><01:21:45.239><c> who</c><01:21:45.480><c> actually</c>\nGPT initially are the ones who actually\nGPT initially are the ones who actually wrote<01:21:46.480><c> Po</c><01:21:47.480><c> and</c><01:21:47.600><c> I</c><01:21:47.760><c> think</c><01:21:47.920><c> they</c><01:21:48.040><c> were</c><01:21:48.280><c> just</c><01:21:48.560><c> like</c>\nwrote Po and I think they were just like\nwrote Po and I think they were just like there<01:21:48.800><c> are</c><01:21:49.000><c> a</c><01:21:49.120><c> lot</c><01:21:49.239><c> of</c><01:21:49.360><c> reinforcement</c>\nthere are a lot of reinforcement\nthere are a lot of reinforcement learning<01:21:50.239><c> people</c><01:21:51.000><c> and</c><01:21:51.120><c> I</c><01:21:51.239><c> think</c><01:21:51.440><c> that</c><01:21:51.600><c> for</c>\nlearning people and I think that for\nlearning people and I think that for them<01:21:51.960><c> it</c><01:21:52.080><c> was</c><01:21:52.280><c> very</c><01:21:52.800><c> intuitive</c><01:21:53.800><c> um</c><01:21:54.760><c> so</c><01:21:55.520><c> there's</c>\nthem it was very intuitive um so there's\nthem it was very intuitive um so there's also<01:21:56.040><c> some</c><01:21:56.400><c> additional</c><01:21:56.960><c> like</c><01:21:57.159><c> potential</c>\nalso some additional like potential\nalso some additional like potential benefits<01:21:58.360><c> for</c><01:21:58.560><c> example</c><01:21:59.560><c> I</c><01:21:59.679><c> don't</c><01:21:59.800><c> want</c><01:22:00.000><c> to</c>\nbenefits for example I don't want to\nbenefits for example I don't want to yeah<01:22:01.159><c> for</c><01:22:01.320><c> example</c><01:22:01.560><c> if</c><01:22:01.639><c> you</c><01:22:01.760><c> use</c><01:22:01.920><c> the</c><01:22:02.040><c> reward</c>\nyeah for example if you use the reward\nyeah for example if you use the reward model<01:22:03.280><c> uh</c><01:22:03.400><c> the</c><01:22:03.520><c> cool</c><01:22:03.760><c> thing</c><01:22:03.920><c> here</c><01:22:04.080><c> with</c>\nmodel uh the cool thing here with\nmodel uh the cool thing here with reinforcement<01:22:04.760><c> learning</c><01:22:05.040><c> is</c><01:22:05.120><c> that</c><01:22:05.239><c> you</c><01:22:05.320><c> can</c>\nreinforcement learning is that you can\nreinforcement learning is that you can use<01:22:05.679><c> unlabeled</c><01:22:06.280><c> data</c><01:22:07.280><c> with</c><01:22:07.400><c> the</c><01:22:07.560><c> reward</c><01:22:07.920><c> model</c>\nuse unlabeled data with the reward model\nuse unlabeled data with the reward model so<01:22:08.480><c> here</c><01:22:08.600><c> you</c><01:22:08.719><c> can</c><01:22:08.920><c> only</c><01:22:09.159><c> use</c><01:22:09.360><c> the</c><01:22:09.520><c> label</c><01:22:09.880><c> data</c>\nso here you can only use the label data\nso here you can only use the label data for<01:22:10.639><c> doing</c><01:22:11.280><c> DPO</c><01:22:12.280><c> um</c><01:22:12.639><c> for</c><01:22:12.920><c> PP</c><01:22:13.360><c> for</c><01:22:13.520><c> po</c><01:22:13.880><c> you</c><01:22:13.960><c> first</c>\nfor doing DPO um for PP for po you first\nfor doing DPO um for PP for po you first train<01:22:14.400><c> your</c><01:22:14.520><c> reward</c><01:22:14.840><c> model</c><01:22:15.520><c> and</c><01:22:15.639><c> then</c><01:22:15.760><c> you</c><01:22:15.880><c> can</c>\ntrain your reward model and then you can\ntrain your reward model and then you can use<01:22:16.239><c> unlabeled</c><01:22:16.800><c> data</c><01:22:17.800><c> uh</c><01:22:18.280><c> where</c><01:22:18.440><c> the</c><01:22:18.600><c> reward</c>\nuse unlabeled data uh where the reward\nuse unlabeled data uh where the reward model<01:22:19.120><c> will</c><01:22:19.320><c> basically</c><01:22:19.719><c> label</c><01:22:20.280><c> this</c>\nmodel will basically label this\nmodel will basically label this unlabeled<01:22:20.960><c> data</c><01:22:21.280><c> so</c><01:22:21.480><c> there</c><01:22:21.719><c> there's</c>\nunlabeled data so there there's\nunlabeled data so there there's additional<01:22:22.360><c> kind</c><01:22:22.520><c> of</c><01:22:23.320><c> potential</c><01:22:24.320><c> uh</c>\nadditional kind of potential uh\nadditional kind of potential uh there<01:22:25.600><c> could</c><01:22:25.760><c> be</c><01:22:25.960><c> potential</c><01:22:26.320><c> improvements</c><01:22:27.080><c> in</c>\nthere could be potential improvements in\nthere could be potential improvements in practice<01:22:27.600><c> it</c><01:22:27.800><c> happens</c><01:22:28.120><c> at</c><01:22:28.400><c> down</c><01:22:28.639><c> and</c><01:22:28.760><c> on</c><01:22:29.360><c> and</c><01:22:29.520><c> I</c>\npractice it happens at down and on and I\npractice it happens at down and on and I think<01:22:30.400><c> just</c><01:22:30.600><c> that</c><01:22:30.760><c> a</c><01:22:30.840><c> lot</c><01:22:30.960><c> of</c><01:22:31.120><c> people</c><01:22:31.679><c> in</c><01:22:31.840><c> this</c>\nthink just that a lot of people in this\nthink just that a lot of people in this team<01:22:32.520><c> were</c><01:22:32.719><c> reinforcement</c><01:22:33.280><c> learning</c><01:22:33.639><c> experts</c>\nteam were reinforcement learning experts\nteam were reinforcement learning experts including<01:22:35.040><c> uh</c><01:22:35.159><c> the</c><01:22:35.239><c> main</c><01:22:35.440><c> author</c><01:22:35.679><c> of</c><01:22:35.800><c> Po</c><01:22:36.360><c> John</c>\nincluding uh the main author of Po John\nincluding uh the main author of Po John hman<01:22:38.880><c> um</c><01:22:39.280><c> so</c><01:22:39.600><c> much</c><01:22:39.800><c> simpler</c><01:22:40.120><c> in</c><01:22:40.239><c> poo</c><01:22:41.000><c> and</c><01:22:41.120><c> is</c>\nhman um so much simpler in poo and is\nhman um so much simpler in poo and is basically<01:22:41.800><c> performs</c><01:22:42.239><c> as</c><01:22:42.400><c> well</c><01:22:43.000><c> uh</c><01:22:43.120><c> so</c><01:22:43.320><c> now</c>\nbasically performs as well uh so now\nbasically performs as well uh so now this<01:22:43.679><c> is</c><01:22:43.800><c> the</c><01:22:43.960><c> standard</c><01:22:44.760><c> uh</c><01:22:44.920><c> thing</c><01:22:45.040><c> that</c>\nthis is the standard uh thing that\nthis is the standard uh thing that people<01:22:45.480><c> use</c><01:22:46.280><c> at</c><01:22:46.440><c> least</c><01:22:46.639><c> in</c><01:22:46.719><c> the</c><01:22:46.840><c> open</c><01:22:47.040><c> source</c>\npeople use at least in the open source\npeople use at least in the open source Community<01:22:47.960><c> I</c><01:22:48.080><c> believe</c><01:22:48.400><c> it's</c><01:22:48.600><c> actually</c><01:22:48.800><c> the</c>\nCommunity I believe it's actually the\nCommunity I believe it's actually the standard<01:22:49.320><c> also</c><01:22:49.520><c> in</c><01:22:50.239><c> in</c><01:22:51.120><c> Industry</c><01:22:52.120><c> so</c><01:22:52.280><c> that's</c>\nstandard also in in Industry so that's\nstandard also in in Industry so that's called<01:22:53.080><c> DPO</c><01:22:54.080><c> gains</c>\ncalled DPO gains\ncalled DPO gains um<01:22:55.320><c> so</c><01:22:55.560><c> those</c><01:22:55.679><c> are</c><01:22:56.080><c> all</c><01:22:56.360><c> the</c><01:22:56.600><c> papers</c><01:22:57.040><c> on</c><01:22:57.159><c> the</c>\num so those are all the papers on the\num so those are all the papers on the left<01:22:57.719><c> here</c><01:22:57.920><c> this</c><01:22:58.000><c> is</c><01:22:58.120><c> on</c><01:22:58.239><c> a</c><01:22:58.360><c> summarization</c>\nleft here this is on a summarization\nleft here this is on a summarization task<01:22:59.639><c> you</c><01:22:59.800><c> see</c><01:23:00.400><c> all</c><01:23:00.560><c> I</c><01:23:00.679><c> want</c><01:23:00.760><c> to</c><01:23:00.920><c> show</c><01:23:01.120><c> you</c><01:23:01.400><c> is</c>\ntask you see all I want to show you is\ntask you see all I want to show you is that<01:23:01.760><c> basically</c><01:23:02.040><c> the</c><01:23:02.159><c> pre-train</c><01:23:02.679><c> models</c><01:23:03.679><c> uh</c>\nthat basically the pre-train models uh\nthat basically the pre-train models uh were<01:23:04.239><c> okay</c><01:23:04.560><c> and</c><01:23:04.679><c> they</c><01:23:04.800><c> improve</c><01:23:05.120><c> with</c><01:23:05.280><c> scale</c><01:23:05.880><c> if</c>\nwere okay and they improve with scale if\nwere okay and they improve with scale if you<01:23:06.080><c> do</c><01:23:06.199><c> supervised</c><01:23:06.679><c> fine</c><01:23:06.880><c> tuning</c><01:23:07.239><c> you</c>\nyou do supervised fine tuning you\nyou do supervised fine tuning you improve<01:23:07.719><c> them</c><01:23:07.880><c> a</c><01:23:07.960><c> little</c><01:23:08.120><c> bit</c><01:23:08.280><c> more</c><01:23:08.760><c> if</c><01:23:08.880><c> you</c><01:23:09.040><c> do</c>\nimprove them a little bit more if you do\nimprove them a little bit more if you do po<01:23:10.040><c> or</c><01:23:10.239><c> something</c><01:23:10.639><c> with</c><01:23:10.800><c> all</c><01:23:11.040><c> HF</c><01:23:11.400><c> with</c><01:23:11.520><c> human</c>\npo or something with all HF with human\npo or something with all HF with human feedback<01:23:12.360><c> you</c><01:23:12.560><c> get</c><01:23:12.880><c> performance</c><01:23:13.320><c> that</c><01:23:13.480><c> are</c><01:23:14.120><c> as</c>\nfeedback you get performance that are as\nfeedback you get performance that are as often<01:23:15.080><c> times</c><01:23:15.719><c> depending</c><01:23:16.040><c> on</c><01:23:16.159><c> a</c><01:23:16.280><c> benchmark</c>\noften times depending on a benchmark\noften times depending on a benchmark even<01:23:17.120><c> better</c><01:23:17.400><c> than</c><01:23:17.960><c> uh</c><01:23:18.120><c> humans</c><01:23:18.520><c> so</c><01:23:18.719><c> this</c><01:23:18.840><c> is</c>\neven better than uh humans so this is\neven better than uh humans so this is the<01:23:19.199><c> human</c><01:23:19.960><c> uh</c><01:23:20.120><c> reference</c><01:23:20.600><c> summaries</c><01:23:21.600><c> same</c>\nthe human uh reference summaries same\nthe human uh reference summaries same thing<01:23:21.960><c> this</c><01:23:22.080><c> is</c><01:23:22.159><c> on</c><01:23:22.320><c> a</c><01:23:22.719><c> uh</c><01:23:22.960><c> on</c><01:23:23.080><c> a</c><01:23:23.199><c> paper</c><01:23:23.480><c> that</c><01:23:23.600><c> we</c>\nthing this is on a uh on a paper that we\nthing this is on a uh on a paper that we have<01:23:23.840><c> Alpaca</c><01:23:24.239><c> Farm</c>\nhave Alpaca Farm\nhave Alpaca Farm where<01:23:25.280><c> we</c><01:23:25.400><c> see</c><01:23:26.199><c> uh</c><01:23:26.320><c> the</c><01:23:26.440><c> evaluation</c><01:23:26.880><c> here</c><01:23:27.000><c> is</c>\nwhere we see uh the evaluation here is\nwhere we see uh the evaluation here is not<01:23:27.199><c> too</c><01:23:27.400><c> important</c><01:23:27.760><c> but</c><01:23:27.920><c> basically</c><01:23:28.199><c> you</c><01:23:28.280><c> see</c>\nnot too important but basically you see\nnot too important but basically you see pre-train<01:23:28.880><c> model</c><01:23:29.440><c> you</c><01:23:29.639><c> jump</c><01:23:29.920><c> to</c><01:23:30.520><c> sft</c><01:23:31.400><c> and</c><01:23:31.520><c> then</c>\npre-train model you jump to sft and then\npre-train model you jump to sft and then you<01:23:31.760><c> jump</c><01:23:31.960><c> to</c><01:23:32.080><c> PPO</c><01:23:32.920><c> and</c><01:23:33.080><c> popo</c><01:23:33.800><c> have</c><01:23:33.960><c> the</c><01:23:34.080><c> exact</c>\nyou jump to PPO and popo have the exact\nyou jump to PPO and popo have the exact same\nsame\nsame performance<01:23:36.840><c> so</c><01:23:37.159><c> basically</c><01:23:37.520><c> all</c><01:23:37.800><c> HF</c><01:23:38.159><c> helps</c>\nperformance so basically all HF helps\nperformance so basically all HF helps that's<01:23:39.040><c> kind</c><01:23:39.159><c> of</c><01:23:39.280><c> the</c><01:23:39.400><c> conclusion</c><01:23:40.120><c> and</c><01:23:40.280><c> DPO</c><01:23:40.719><c> is</c>\nthat's kind of the conclusion and DPO is\nthat's kind of the conclusion and DPO is simple<01:23:42.400><c> uh</c><01:23:42.600><c> data</c><01:23:43.400><c> uh</c><01:23:43.560><c> the</c><01:23:43.800><c> way</c><01:23:44.040><c> that</c><01:23:44.159><c> you</c>\nsimple uh data uh the way that you\nsimple uh data uh the way that you collect<01:23:44.719><c> that</c><01:23:44.880><c> type</c><01:23:45.040><c> of</c><01:23:45.239><c> data</c><01:23:46.120><c> um</c><01:23:47.080><c> first</c><01:23:47.400><c> idea</c>\ncollect that type of data um first idea\ncollect that type of data um first idea is<01:23:48.080><c> just</c><01:23:48.239><c> use</c><01:23:48.679><c> humans</c><01:23:49.239><c> as</c><01:23:49.360><c> we</c><01:23:49.520><c> already</c><01:23:49.760><c> talked</c>\nis just use humans as we already talked\nis just use humans as we already talked about<01:23:50.840><c> uh</c><01:23:51.080><c> guidelines</c><01:23:51.480><c> are</c><01:23:51.679><c> very</c><01:23:51.880><c> complicated</c>\nabout uh guidelines are very complicated\nabout uh guidelines are very complicated for<01:23:52.920><c> what</c><01:23:53.080><c> humans</c><01:23:53.360><c> should</c><01:23:53.520><c> be</c><01:23:53.639><c> labeling</c><01:23:54.280><c> and</c>\nfor what humans should be labeling and\nfor what humans should be labeling and and<01:23:54.480><c> it's</c><01:23:54.639><c> really</c><01:23:54.840><c> not</c><01:23:55.040><c> that</c><01:23:55.199><c> easy</c><01:23:55.679><c> and</c>\nand it's really not that easy and\nand it's really not that easy and actually<01:23:56.120><c> if</c><01:23:56.199><c> you</c><01:23:56.400><c> ever</c><01:23:56.639><c> do</c><01:23:56.920><c> some</c><01:23:57.120><c> of</c><01:23:57.239><c> the</c>\nactually if you ever do some of the\nactually if you ever do some of the labeling<01:23:58.159><c> you</c><01:23:58.239><c> will</c><01:23:58.480><c> see</c><01:23:58.840><c> that</c><01:23:59.480><c> it's</c>\nlabeling you will see that it's\nlabeling you will see that it's extremely<01:24:00.639><c> complicated</c><01:24:01.400><c> like</c><01:24:01.520><c> if</c><01:24:01.639><c> I</c><01:24:01.800><c> zoom</c><01:24:02.080><c> in</c>\nextremely complicated like if I zoom in\nextremely complicated like if I zoom in to<01:24:02.719><c> this</c><01:24:03.719><c> uh</c><01:24:03.960><c> here</c><01:24:04.120><c> I</c><01:24:04.239><c> have</c><01:24:04.360><c> a</c><01:24:04.520><c> question</c><01:24:05.080><c> tell</c>\nto this uh here I have a question tell\nto this uh here I have a question tell tell<01:24:06.000><c> me</c><01:24:06.239><c> about</c><01:24:06.440><c> self-driving</c><01:24:07.000><c> cars</c><01:24:07.719><c> and</c><01:24:07.800><c> you</c>\ntell me about self-driving cars and you\ntell me about self-driving cars and you read<01:24:08.159><c> both</c><01:24:08.360><c> self-driving</c><01:24:08.880><c> cars</c><01:24:09.080><c> are</c><01:24:09.239><c> vehicles</c>\nread both self-driving cars are vehicles\nread both self-driving cars are vehicles that<01:24:09.679><c> are</c><01:24:09.800><c> capable</c><01:24:10.080><c> of</c><01:24:10.199><c> detecting</c><01:24:10.600><c> their</c>\nthat are capable of detecting their\nthat are capable of detecting their surroundings<01:24:11.239><c> blah</c><01:24:11.400><c> blah</c><01:24:11.600><c> blah</c><01:24:11.960><c> self-driving</c>\nsurroundings blah blah blah self-driving\nsurroundings blah blah blah self-driving cars<01:24:12.719><c> are</c><01:24:12.920><c> cars</c><01:24:13.120><c> that</c><01:24:13.239><c> are</c><01:24:13.360><c> equipped</c><01:24:13.679><c> with</c>\ncars are cars that are equipped with\ncars are cars that are equipped with sensors<01:24:14.199><c> blah</c><01:24:14.400><c> blah</c><01:24:14.600><c> blah</c><01:24:14.880><c> to</c><01:24:15.040><c> navigate</c>\nsensors blah blah blah to navigate\nsensors blah blah blah to navigate without<01:24:15.679><c> the</c><01:24:15.800><c> need</c><01:24:15.920><c> for</c><01:24:16.080><c> a</c><01:24:16.199><c> driver</c><01:24:16.679><c> I</c><01:24:16.760><c> mean</c>\nwithout the need for a driver I mean\nwithout the need for a driver I mean both<01:24:17.199><c> seem</c><01:24:17.639><c> okay</c><01:24:18.120><c> like</c><01:24:18.280><c> which</c><01:24:18.440><c> one</c><01:24:18.600><c> is</c><01:24:18.719><c> better</c>\nboth seem okay like which one is better\nboth seem okay like which one is better it's<01:24:19.400><c> actually</c><01:24:19.600><c> hard</c><01:24:19.800><c> to</c><01:24:19.960><c> say</c><01:24:20.280><c> at</c><01:24:20.400><c> a</c><01:24:20.600><c> glance</c><01:24:21.400><c> um</c>\nit's actually hard to say at a glance um\nit's actually hard to say at a glance um and<01:24:22.159><c> as</c><01:24:22.239><c> a</c><01:24:22.440><c> result</c><01:24:23.080><c> uh</c><01:24:23.199><c> the</c><01:24:23.400><c> problem</c><01:24:23.639><c> with</c>\nand as a result uh the problem with\nand as a result uh the problem with humans<01:24:24.639><c> is</c><01:24:24.920><c> that</c><01:24:25.840><c> you</c><01:24:25.960><c> will</c><01:24:26.159><c> start</c><01:24:26.520><c> optimizing</c>\nhumans is that you will start optimizing\nhumans is that you will start optimizing a<01:24:27.199><c> lot</c><01:24:27.320><c> of</c><01:24:27.480><c> like</c><01:24:27.639><c> high</c><01:24:27.840><c> level</c><01:24:28.080><c> features</c><01:24:28.520><c> for</c>\na lot of like high level features for\na lot of like high level features for example<01:24:29.000><c> the</c><01:24:29.080><c> second</c><01:24:29.320><c> one</c><01:24:29.440><c> is</c><01:24:29.600><c> longer</c><01:24:30.199><c> I</c><01:24:30.320><c> can</c>\nexample the second one is longer I can\nexample the second one is longer I can guarantee<01:24:31.120><c> you</c><01:24:31.280><c> that</c><01:24:31.400><c> most</c><01:24:31.560><c> humans</c><01:24:31.840><c> will</c>\nguarantee you that most humans will\nguarantee you that most humans will choose<01:24:32.520><c> second</c><01:24:32.800><c> one</c><01:24:33.400><c> even</c><01:24:33.719><c> though</c><01:24:34.440><c> I</c><01:24:34.480><c> mean</c>\nchoose second one even though I mean\nchoose second one even though I mean maybe<01:24:34.840><c> the</c><01:24:34.960><c> first</c><01:24:35.159><c> one</c><01:24:35.280><c> is</c><01:24:35.400><c> better</c><01:24:35.639><c> I</c><01:24:35.679><c> don't</c>\nmaybe the first one is better I don't\nmaybe the first one is better I don't know<01:24:36.320><c> I</c><01:24:36.440><c> haven't</c><01:24:36.719><c> read</c><01:24:36.880><c> it</c><01:24:37.520><c> carefully</c><01:24:38.520><c> so</c>\nknow I haven't read it carefully so\nknow I haven't read it carefully so challenges<01:24:39.080><c> with</c><01:24:39.280><c> humans</c><01:24:40.000><c> first</c><01:24:40.480><c> slow</c><01:24:40.760><c> and</c>\nchallenges with humans first slow and\nchallenges with humans first slow and expensive<01:24:42.360><c> uh</c><01:24:42.719><c> second</c><01:24:43.360><c> as</c><01:24:43.520><c> I</c><01:24:43.679><c> just</c><01:24:43.880><c> mentioned</c>\nexpensive uh second as I just mentioned\nexpensive uh second as I just mentioned it's<01:24:44.760><c> hard</c><01:24:44.960><c> to</c><01:24:45.199><c> focus</c><01:24:45.639><c> on</c><01:24:45.840><c> things</c><01:24:46.040><c> that</c><01:24:46.199><c> matter</c>\nit's hard to focus on things that matter\nit's hard to focus on things that matter like<01:24:46.719><c> correctness</c><01:24:47.440><c> and</c><01:24:47.639><c> people</c><01:24:48.480><c> uh</c><01:24:48.639><c> usually</c>\nlike correctness and people uh usually\nlike correctness and people uh usually look<01:24:49.199><c> at</c><01:24:49.400><c> things</c><01:24:49.639><c> that</c><01:24:50.239><c> don't</c><01:24:50.480><c> matter</c><01:24:50.760><c> as</c><01:24:50.880><c> much</c>\nlook at things that don't matter as much\nlook at things that don't matter as much like<01:24:51.320><c> the</c><01:24:51.480><c> form</c><01:24:51.920><c> like</c><01:24:52.520><c> length</c><01:24:53.520><c> uh</c><01:24:53.639><c> and</c><01:24:53.760><c> as</c><01:24:53.840><c> a</c>\nlike the form like length uh and as a\nlike the form like length uh and as a result<01:24:54.480><c> so</c><01:24:54.639><c> what</c><01:24:54.719><c> I</c><01:24:54.840><c> show</c><01:24:55.080><c> here</c><01:24:55.239><c> is</c><01:24:55.360><c> that</c><01:24:55.760><c> uh</c>\nresult so what I show here is that uh\nresult so what I show here is that uh when<01:24:56.000><c> you</c><01:24:56.080><c> do</c><01:24:56.239><c> lhf</c><01:24:57.000><c> the</c><01:24:57.080><c> more</c><01:24:57.239><c> you</c><01:24:57.360><c> do</c><01:24:57.600><c> of</c><01:24:57.760><c> lhf</c>\nwhen you do lhf the more you do of lhf\nwhen you do lhf the more you do of lhf the<01:24:58.480><c> longer</c><01:24:58.760><c> the</c><01:24:58.840><c> output</c><01:24:59.159><c> of</c><01:24:59.320><c> the</c><01:24:59.679><c> of</c><01:24:59.960><c> the</c>\nthe longer the output of the of the\nthe longer the output of the of the models<01:25:00.920><c> become</c><01:25:01.400><c> so</c><01:25:01.560><c> if</c><01:25:01.639><c> you've</c><01:25:01.800><c> ever</c><01:25:01.960><c> been</c>\nmodels become so if you've ever been\nmodels become so if you've ever been annoyed<01:25:02.760><c> at</c><01:25:02.960><c> chat</c><01:25:03.159><c> GPT</c><01:25:03.600><c> answering</c><01:25:04.000><c> you</c><01:25:04.280><c> super</c>\nannoyed at chat GPT answering you super\nannoyed at chat GPT answering you super long<01:25:04.719><c> sentences</c><01:25:05.400><c> this</c><01:25:05.520><c> is</c><01:25:05.679><c> because</c><01:25:05.840><c> of</c><01:25:06.000><c> all</c>\nlong sentences this is because of all\nlong sentences this is because of all rhf<01:25:08.159><c> um</c><01:25:08.639><c> annotator</c><01:25:09.239><c> distribution</c><01:25:09.719><c> shift</c><01:25:10.600><c> uh</c>\nrhf um annotator distribution shift uh\nrhf um annotator distribution shift uh like<01:25:11.760><c> the</c><01:25:11.880><c> distribution</c><01:25:12.360><c> of</c><01:25:12.480><c> annotators</c><01:25:13.000><c> that</c>\nlike the distribution of annotators that\nlike the distribution of annotators that you<01:25:13.239><c> use</c><01:25:13.800><c> matters</c><01:25:14.159><c> a</c><01:25:14.360><c> lot</c><01:25:14.960><c> and</c><01:25:15.040><c> you</c><01:25:15.199><c> have</c><01:25:15.320><c> to</c>\nyou use matters a lot and you have to\nyou use matters a lot and you have to think<01:25:15.800><c> like</c><01:25:16.080><c> what</c><01:25:16.239><c> is</c><01:25:16.719><c> what</c><01:25:16.840><c> is</c><01:25:17.000><c> even</c><01:25:17.159><c> the</c>\nthink like what is what is even the\nthink like what is what is even the humans<01:25:17.600><c> that</c><01:25:17.719><c> we</c><01:25:17.800><c> want</c><01:25:17.920><c> to</c><01:25:18.040><c> represent</c><01:25:18.679><c> in</c>\nhumans that we want to represent in\nhumans that we want to represent in these<01:25:19.000><c> models</c><01:25:20.000><c> uh</c><01:25:20.119><c> now</c><01:25:20.320><c> the</c><01:25:20.440><c> question</c><01:25:20.679><c> is</c><01:25:20.800><c> like</c>\nthese models uh now the question is like\nthese models uh now the question is like crowdsourcing<01:25:21.600><c> ethics</c><01:25:22.520><c> uh</c><01:25:23.119><c> like</c><01:25:23.480><c> usually</c>\ncrowdsourcing ethics uh like usually\ncrowdsourcing ethics uh like usually these<01:25:24.560><c> basically</c><01:25:24.880><c> a</c><01:25:25.000><c> lot</c><01:25:25.119><c> of</c><01:25:25.320><c> the</c><01:25:25.480><c> the</c>\nthese basically a lot of the the\nthese basically a lot of the the labeling<01:25:26.000><c> that</c><01:25:26.119><c> is</c><01:25:26.280><c> done</c><01:25:27.080><c> um</c><01:25:28.000><c> like</c><01:25:28.440><c> the</c><01:25:28.600><c> people</c>\nlabeling that is done um like the people\nlabeling that is done um like the people who<01:25:29.040><c> do</c><01:25:29.320><c> them</c><01:25:29.520><c> are</c><01:25:29.679><c> not</c><01:25:29.960><c> paid</c><01:25:30.239><c> well</c><01:25:30.600><c> and</c><01:25:30.719><c> they</c>\nwho do them are not paid well and they\nwho do them are not paid well and they have<01:25:30.960><c> to</c><01:25:31.119><c> go</c><01:25:31.239><c> through</c><01:25:31.400><c> a</c><01:25:31.520><c> lot</c><01:25:31.639><c> of</c><01:25:31.760><c> toxic</c><01:25:32.080><c> data</c>\nhave to go through a lot of toxic data\nhave to go through a lot of toxic data uh<01:25:33.159><c> because</c><01:25:33.360><c> you</c><01:25:33.520><c> basically</c><01:25:33.880><c> want</c><01:25:34.119><c> the</c><01:25:34.239><c> model</c>\nuh because you basically want the model\nuh because you basically want the model to<01:25:34.719><c> avoid</c><01:25:34.960><c> saying</c><01:25:35.320><c> the</c><01:25:35.480><c> toxic</c><01:25:35.840><c> data</c><01:25:36.719><c> um</c><01:25:37.080><c> so</c>\nto avoid saying the toxic data um so\nto avoid saying the toxic data um so crowdsourcing<01:25:38.320><c> ethics</c>\ncrowdsourcing ethics\ncrowdsourcing ethics too<01:25:40.360><c> so</c><01:25:40.679><c> many</c><01:25:41.000><c> challenges</c><01:25:41.440><c> with</c><01:25:41.560><c> human</c><01:25:41.840><c> data</c>\ntoo so many challenges with human data\ntoo so many challenges with human data um<01:25:43.280><c> so</c><01:25:43.600><c> what</c><01:25:43.760><c> we</c><01:25:43.960><c> did</c><01:25:44.360><c> also</c><01:25:44.719><c> last</c><01:25:44.920><c> year</c><01:25:45.400><c> is</c>\num so what we did also last year is\num so what we did also last year is again<01:25:45.960><c> the</c><01:25:46.080><c> same</c><01:25:46.280><c> thing</c><01:25:46.400><c> as</c><01:25:46.560><c> alpaca</c><01:25:47.159><c> just</c><01:25:47.320><c> the</c>\nagain the same thing as alpaca just the\nagain the same thing as alpaca just the idea<01:25:47.760><c> of</c><01:25:47.960><c> like</c><01:25:48.119><c> oh</c><01:25:48.320><c> well</c><01:25:48.560><c> they're</c><01:25:48.760><c> challenges</c>\nidea of like oh well they're challenges\nidea of like oh well they're challenges with<01:25:49.239><c> humans</c><01:25:49.520><c> maybe</c><01:25:49.679><c> we</c><01:25:49.760><c> can</c><01:25:49.880><c> just</c><01:25:50.000><c> replace</c>\nwith humans maybe we can just replace\nwith humans maybe we can just replace them<01:25:50.480><c> with</c><01:25:50.880><c> llms</c><01:25:51.880><c> uh</c><01:25:52.080><c> so</c><01:25:52.320><c> what</c><01:25:52.440><c> we</c><01:25:52.600><c> did</c><01:25:52.800><c> is</c>\nthem with llms uh so what we did is\nthem with llms uh so what we did is simply<01:25:53.440><c> replace</c>\nsimply replace\nsimply replace um<01:25:55.480><c> oh</c><01:25:55.679><c> I</c><01:25:55.840><c> see</c><01:25:56.159><c> that</c><01:25:56.880><c> I'm</c><01:25:57.000><c> just</c><01:25:57.159><c> realizing</c><01:25:57.560><c> that</c>\num oh I see that I'm just realizing that\num oh I see that I'm just realizing that the<01:25:57.840><c> slides</c><01:25:58.080><c> are</c><01:25:58.199><c> not</c><01:25:58.360><c> sented</c><01:25:58.920><c> anyways</c><01:25:59.679><c> uh</c><01:25:59.800><c> you</c>\nthe slides are not sented anyways uh you\nthe slides are not sented anyways uh you replace<01:26:00.520><c> a</c><01:26:00.639><c> human</c><01:26:00.920><c> preference</c><01:26:01.320><c> with</c><01:26:01.440><c> LM</c>\nreplace a human preference with LM\nreplace a human preference with LM preferences<01:26:02.760><c> uh</c><01:26:02.880><c> so</c><01:26:03.199><c> here</c><01:26:03.560><c> on</c><01:26:03.840><c> this</c><01:26:04.360><c> uh</c><01:26:04.520><c> figure</c>\npreferences uh so here on this uh figure\npreferences uh so here on this uh figure you<01:26:04.920><c> see</c><01:26:05.080><c> on</c><01:26:05.199><c> the</c><01:26:05.320><c> xaxis</c><01:26:05.880><c> the</c><01:26:06.040><c> price</c><01:26:06.600><c> that</c><01:26:06.760><c> we</c>\nyou see on the xaxis the price that we\nyou see on the xaxis the price that we paid<01:26:07.880><c> uh</c><01:26:07.960><c> for</c><01:26:08.199><c> collecting</c><01:26:08.639><c> human</c><01:26:08.920><c> data</c><01:26:09.239><c> it's</c>\npaid uh for collecting human data it's\npaid uh for collecting human data it's around\naround\naround $300<01:26:11.320><c> for</c><01:26:11.560><c> 1,000</c><01:26:12.080><c> examples</c><01:26:12.600><c> and</c><01:26:12.719><c> this</c><01:26:12.840><c> is</c><01:26:13.000><c> on</c>\n$300 for 1,000 examples and this is on\n$300 for 1,000 examples and this is on mechanical<01:26:13.719><c> turkers</c><01:26:14.239><c> which</c><01:26:14.360><c> are</c><01:26:15.159><c> usually</c>\nmechanical turkers which are usually\nmechanical turkers which are usually like<01:26:15.760><c> cheaper</c><01:26:16.239><c> than</c><01:26:16.560><c> than</c><01:26:16.880><c> maybe</c><01:26:17.119><c> some</c><01:26:17.239><c> of</c><01:26:17.360><c> the</c>\nlike cheaper than than maybe some of the\nlike cheaper than than maybe some of the other<01:26:18.920><c> um</c><01:26:19.320><c> companies</c><01:26:19.679><c> that</c><01:26:19.840><c> you</c><01:26:19.920><c> could</c><01:26:20.040><c> go</c>\nother um companies that you could go\nother um companies that you could go through<01:26:20.760><c> and</c><01:26:20.920><c> on</c><01:26:21.040><c> the</c><01:26:21.480><c> Y</c><01:26:21.719><c> AIS</c><01:26:22.280><c> it's</c><01:26:22.480><c> basically</c>\nthrough and on the Y AIS it's basically\nthrough and on the Y AIS it's basically the<01:26:23.000><c> agreement</c><01:26:23.920><c> with</c><01:26:24.520><c> uh</c><01:26:24.679><c> other</c><01:26:24.920><c> humans</c><01:26:25.360><c> with</c>\nthe agreement with uh other humans with\nthe agreement with uh other humans with the<01:26:25.639><c> mode</c><01:26:25.920><c> of</c><01:26:26.119><c> other</c><01:26:26.360><c> humans</c><01:26:27.119><c> and</c><01:26:27.280><c> what</c><01:26:27.400><c> you</c>\nthe mode of other humans and what you\nthe mode of other humans and what you see<01:26:27.679><c> is</c><01:26:27.800><c> that</c><01:26:28.000><c> actually</c><01:26:28.400><c> as</c><01:26:28.520><c> I</c><01:26:28.600><c> told</c><01:26:28.800><c> you</c>\nsee is that actually as I told you\nsee is that actually as I told you before<01:26:29.199><c> labeling</c><01:26:29.600><c> is</c><01:26:29.760><c> really</c><01:26:29.960><c> complicated</c>\nbefore labeling is really complicated\nbefore labeling is really complicated humans<01:26:31.159><c> agree</c><01:26:31.560><c> with</c><01:26:31.920><c> themselves</c><01:26:32.639><c> only</c><01:26:32.960><c> around</c>\nhumans agree with themselves only around\nhumans agree with themselves only around 66%<01:26:34.000><c> of</c><01:26:34.119><c> the</c><01:26:34.239><c> time</c><01:26:34.880><c> on</c><01:26:35.000><c> a</c><01:26:35.119><c> binary</c><01:26:35.520><c> Tas</c><01:26:36.199><c> and</c><01:26:36.400><c> it's</c>\n66% of the time on a binary Tas and it's\n66% of the time on a binary Tas and it's not<01:26:36.880><c> that</c><01:26:37.000><c> the</c><01:26:37.239><c> humans</c><01:26:37.520><c> are</c><01:26:37.679><c> not</c><01:26:37.880><c> good</c><01:26:38.159><c> here</c>\nnot that the humans are not good here\nnot that the humans are not good here because<01:26:39.239><c> uh</c><01:26:39.360><c> we</c><01:26:39.440><c> were</c><01:26:39.760><c> five</c><01:26:40.040><c> main</c><01:26:40.320><c> authors</c><01:26:40.639><c> on</c>\nbecause uh we were five main authors on\nbecause uh we were five main authors on this<01:26:40.960><c> paper</c><01:26:41.480><c> we</c><01:26:41.639><c> tried</c><01:26:41.920><c> to</c><01:26:42.080><c> label</c><01:26:42.960><c> this</c><01:26:43.159><c> data</c>\nthis paper we tried to label this data\nthis paper we tried to label this data ourselves<01:26:44.159><c> and</c><01:26:44.320><c> we</c><01:26:44.480><c> only</c><01:26:44.760><c> had</c><01:26:44.960><c> like</c><01:26:45.199><c> say</c><01:26:45.480><c> 67</c><01:26:46.080><c> or</c>\nourselves and we only had like say 67 or\nourselves and we only had like say 67 or 68%<01:26:47.040><c> accuracy</c><01:26:47.719><c> even</c><01:26:47.960><c> though</c><01:26:48.119><c> we</c><01:26:48.400><c> talk</c><01:26:48.679><c> like</c><01:26:48.760><c> we</c>\n68% accuracy even though we talk like we\n68% accuracy even though we talk like we talk<01:26:49.080><c> for</c><01:26:49.199><c> like</c><01:26:49.360><c> 3</c><01:26:49.560><c> hours</c><01:26:49.840><c> of</c><01:26:50.000><c> how</c><01:26:50.119><c> we</c><01:26:50.199><c> should</c>\ntalk for like 3 hours of how we should\ntalk for like 3 hours of how we should be<01:26:50.520><c> doing</c><01:26:50.760><c> labeling</c><01:26:51.600><c> really</c><01:26:51.760><c> it's</c>\nbe doing labeling really it's\nbe doing labeling really it's complicated<01:26:52.679><c> it's</c><01:26:52.840><c> not</c><01:26:53.000><c> an</c><01:26:53.159><c> easy</c><01:26:53.440><c> task</c><01:26:54.119><c> um</c><01:26:54.440><c> and</c>\ncomplicated it's not an easy task um and\ncomplicated it's not an easy task um and here<01:26:54.760><c> I</c><01:26:54.880><c> just</c><01:26:55.000><c> showed</c><01:26:55.280><c> many</c><01:26:55.480><c> different</c><01:26:55.719><c> models</c>\nhere I just showed many different models\nhere I just showed many different models and<01:26:56.880><c> um</c><01:26:57.280><c> basically</c><01:26:57.639><c> you</c><01:26:57.760><c> see</c><01:26:57.960><c> that</c><01:26:58.119><c> models</c><01:26:58.400><c> are</c>\nand um basically you see that models are\nand um basically you see that models are much<01:26:58.760><c> cheaper</c><01:26:59.400><c> and</c><01:26:59.520><c> they</c><01:26:59.679><c> can</c><01:26:59.840><c> actually</c><01:27:00.119><c> get</c>\nmuch cheaper and they can actually get\nmuch cheaper and they can actually get higher<01:27:01.080><c> agreement</c><01:27:01.560><c> with</c><01:27:01.880><c> the</c><01:27:01.960><c> mode</c><01:27:02.239><c> of</c><01:27:02.360><c> humans</c>\nhigher agreement with the mode of humans\nhigher agreement with the mode of humans than<01:27:03.119><c> human</c><01:27:03.600><c> humans</c><01:27:04.080><c> themselves</c><01:27:04.679><c> and</c><01:27:04.800><c> the</c>\nthan human humans themselves and the\nthan human humans themselves and the reason<01:27:05.199><c> why</c><01:27:05.360><c> is</c><01:27:05.480><c> because</c><01:27:05.639><c> humans</c><01:27:05.920><c> have</c><01:27:06.040><c> a</c><01:27:06.119><c> lot</c>\nreason why is because humans have a lot\nreason why is because humans have a lot of<01:27:06.320><c> varant</c><01:27:06.880><c> models</c><01:27:07.239><c> have</c><01:27:07.360><c> no</c><01:27:07.520><c> varant</c><01:27:07.920><c> so</c><01:27:08.080><c> they</c>\nof varant models have no varant so they\nof varant models have no varant so they might<01:27:08.320><c> be</c><01:27:08.400><c> a</c><01:27:08.480><c> little</c><01:27:08.600><c> bit</c><01:27:08.760><c> more</c><01:27:08.920><c> biased</c><01:27:09.560><c> but</c>\nmight be a little bit more biased but\nmight be a little bit more biased but have<01:27:09.920><c> less</c><01:27:10.280><c> virence</c><01:27:11.280><c> uh</c><01:27:11.639><c> so</c><01:27:11.800><c> it</c><01:27:11.920><c> works</c>\nhave less virence uh so it works\nhave less virence uh so it works surprisingly<01:27:12.760><c> well</c><01:27:13.440><c> and</c><01:27:13.639><c> now</c><01:27:13.840><c> it's</c><01:27:14.040><c> kind</c><01:27:14.159><c> of</c>\nsurprisingly well and now it's kind of\nsurprisingly well and now it's kind of the<01:27:14.480><c> standard</c><01:27:14.840><c> in</c><01:27:15.119><c> open</c><01:27:15.679><c> uh</c><01:27:15.840><c> Source</c><01:27:16.159><c> Community</c>\nthe standard in open uh Source Community\nthe standard in open uh Source Community I<01:27:16.800><c> think</c><01:27:16.960><c> even</c><01:27:17.119><c> in</c><01:27:17.400><c> Industry</c><01:27:18.199><c> a</c><01:27:18.360><c> lot</c><01:27:18.480><c> of</c><01:27:18.639><c> people</c>\nI think even in Industry a lot of people\nI think even in Industry a lot of people use<01:27:19.199><c> both</c><01:27:19.440><c> humans</c><01:27:19.840><c> and</c><01:27:20.000><c> llms</c><01:27:20.760><c> for</c><01:27:21.000><c> improving</c>\nuse both humans and llms for improving\nuse both humans and llms for improving uh<01:27:21.840><c> the</c><01:27:21.960><c> colle</c><01:27:22.440><c> collection</c><01:27:22.760><c> of</c><01:27:22.880><c> allf</c><01:27:23.400><c> data</c>\nuh the colle collection of allf data\nuh the colle collection of allf data um<01:27:25.159><c> and</c><01:27:25.360><c> this</c><01:27:25.480><c> is</c><01:27:25.679><c> like</c><01:27:25.880><c> this</c><01:27:25.960><c> is</c><01:27:26.159><c> the</c><01:27:26.280><c> paper</c>\num and this is like this is the paper\num and this is like this is the paper from<01:27:26.760><c> last</c><01:27:26.960><c> year</c><01:27:27.199><c> but</c><01:27:27.400><c> honestly</c><01:27:27.840><c> now</c><01:27:28.000><c> it's</c>\nfrom last year but honestly now it's\nfrom last year but honestly now it's more<01:27:28.400><c> like</c><01:27:29.280><c> that</c><01:27:29.480><c> llms</c><01:27:29.920><c> would</c><01:27:30.080><c> be</c><01:27:30.239><c> around</c><01:27:30.560><c> this</c>\nmore like that llms would be around this\nmore like that llms would be around this agreement<01:27:31.320><c> and</c><01:27:31.480><c> this</c><01:27:31.639><c> cost</c><01:27:31.880><c> so</c><01:27:32.000><c> around</c><01:27:32.560><c> I</c>\nagreement and this cost so around I\nagreement and this cost so around I would<01:27:32.800><c> say</c><01:27:32.960><c> 50x</c><01:27:33.520><c> cheaper</c><01:27:33.880><c> than</c><01:27:34.080><c> humans</c><01:27:34.719><c> and</c>\nwould say 50x cheaper than humans and\nwould say 50x cheaper than humans and better<01:27:35.159><c> agreement</c><01:27:35.560><c> with</c><01:27:35.719><c> human</c><01:27:36.440><c> than</c><01:27:36.679><c> humans</c>\nbetter agreement with human than humans\nbetter agreement with human than humans themselves<01:27:39.199><c> okay</c><01:27:39.960><c> so</c><01:27:40.960><c> that</c><01:27:41.199><c> gets</c><01:27:41.440><c> us</c><01:27:41.600><c> to</c>\nthemselves okay so that gets us to\nthemselves okay so that gets us to evaluation<01:27:42.320><c> of</c><01:27:42.520><c> post</c>\nevaluation of post\nevaluation of post training<01:27:44.480><c> um</c><01:27:45.199><c> that</c><01:27:45.360><c> goes</c><01:27:45.600><c> back</c><01:27:45.760><c> to</c><01:27:45.920><c> your</c>\ntraining um that goes back to your\ntraining um that goes back to your initial<01:27:46.520><c> question</c><01:27:46.760><c> at</c><01:27:46.880><c> the</c><01:27:46.960><c> beginning</c><01:27:47.199><c> of</c><01:27:47.280><c> the</c>\ninitial question at the beginning of the\ninitial question at the beginning of the lecture<01:27:47.960><c> how</c><01:27:48.159><c> do</c><01:27:48.239><c> you</c><01:27:48.400><c> evaluate</c><01:27:48.760><c> something</c>\nlecture how do you evaluate something\nlecture how do you evaluate something like<01:27:49.280><c> chpt</c><01:27:50.280><c> uh</c><01:27:50.400><c> the</c><01:27:50.600><c> answers</c><01:27:50.880><c> that</c><01:27:51.000><c> chpt</c><01:27:51.560><c> could</c>\nlike chpt uh the answers that chpt could\nlike chpt uh the answers that chpt could give<01:27:52.560><c> are</c><01:27:52.880><c> basically</c><01:27:53.360><c> unbounded</c><01:27:54.560><c> and</c><01:27:54.719><c> it's</c>\ngive are basically unbounded and it's\ngive are basically unbounded and it's not<01:27:55.080><c> that</c><01:27:55.199><c> there</c><01:27:55.440><c> one</c><01:27:55.679><c> right</c><01:27:55.920><c> answer</c><01:27:56.360><c> there</c>\nnot that there one right answer there\nnot that there one right answer there are<01:27:56.639><c> many</c><01:27:56.920><c> answers</c><01:27:57.320><c> that</c><01:27:57.440><c> are</c><01:27:57.639><c> just</c><01:27:57.800><c> as</c><01:27:58.040><c> good</c>\nare many answers that are just as good\nare many answers that are just as good um<01:27:59.280><c> so</c><01:27:59.440><c> there</c><01:27:59.520><c> are</c><01:27:59.639><c> many</c><01:27:59.840><c> challenges</c><01:28:00.560><c> one</c><01:28:01.119><c> you</c>\num so there are many challenges one you\num so there are many challenges one you can't<01:28:01.719><c> use</c><01:28:02.400><c> validation</c><01:28:02.840><c> loss</c><01:28:03.639><c> because</c><01:28:04.600><c> one</c>\ncan't use validation loss because one\ncan't use validation loss because one method<01:28:05.280><c> might</c><01:28:05.400><c> use</c><01:28:05.600><c> po</c><01:28:06.000><c> the</c><01:28:06.119><c> other</c><01:28:06.239><c> one</c><01:28:06.400><c> might</c>\nmethod might use po the other one might\nmethod might use po the other one might use<01:28:06.679><c> DPO</c><01:28:07.280><c> validation</c><01:28:07.679><c> loss</c><01:28:07.960><c> is</c><01:28:08.080><c> not</c>\nuse DPO validation loss is not\nuse DPO validation loss is not comparable<01:28:09.159><c> second</c><01:28:09.520><c> you</c><01:28:09.679><c> can't</c><01:28:09.840><c> use</c><01:28:10.159><c> Cal</c><01:28:10.639><c> uh</c>\ncomparable second you can't use Cal uh\ncomparable second you can't use Cal uh sorry<01:28:11.080><c> perplexity</c><01:28:11.880><c> that's</c><01:28:12.000><c> the</c><01:28:12.159><c> thing</c><01:28:12.280><c> I</c><01:28:12.400><c> told</c>\nsorry perplexity that's the thing I told\nsorry perplexity that's the thing I told you<01:28:12.719><c> before</c><01:28:13.480><c> these</c><01:28:13.760><c> models</c><01:28:14.760><c> uh</c><01:28:14.920><c> are</c><01:28:15.080><c> not</c>\nyou before these models uh are not\nyou before these models uh are not calibrated<01:28:15.920><c> they</c><01:28:16.000><c> don't</c><01:28:16.239><c> give</c><01:28:16.679><c> distributions</c>\ncalibrated they don't give distributions\ncalibrated they don't give distributions they<01:28:17.840><c> they</c><01:28:17.960><c> just</c><01:28:18.239><c> optimize</c><01:28:18.719><c> for</c><01:28:18.920><c> one</c><01:28:19.119><c> thing</c><01:28:19.400><c> so</c>\nthey they just optimize for one thing so\nthey they just optimize for one thing so you<01:28:19.639><c> can't</c><01:28:19.840><c> use</c><01:28:20.040><c> perplexity</c><01:28:20.560><c> for</c><01:28:20.800><c> actually</c>\nyou can't use perplexity for actually\nyou can't use perplexity for actually evaluating<01:28:22.119><c> uh</c><01:28:22.239><c> these</c><01:28:22.400><c> type</c><01:28:22.600><c> of</c><01:28:22.679><c> models</c><01:28:23.040><c> once</c>\nevaluating uh these type of models once\nevaluating uh these type of models once they're<01:28:23.400><c> aligned</c><01:28:24.400><c> sorry</c><01:28:24.719><c> one</c><01:28:24.960><c> Z</c><01:28:25.679><c> lined</c><01:28:26.679><c> third</c>\nthey're aligned sorry one Z lined third\nthey're aligned sorry one Z lined third uh<01:28:27.639><c> there's</c><01:28:27.760><c> a</c><01:28:27.880><c> large</c><01:28:28.199><c> diversity</c><01:28:28.639><c> of</c>\nuh there's a large diversity of\nuh there's a large diversity of questions<01:28:29.040><c> that</c><01:28:29.199><c> human</c><01:28:29.520><c> might</c><01:28:29.840><c> ask</c><01:28:30.159><c> to</c><01:28:30.360><c> these</c>\nquestions that human might ask to these\nquestions that human might ask to these models<01:28:31.119><c> generation</c><01:28:31.800><c> open</c><01:28:32.159><c> QA</c><01:28:32.719><c> like</c><01:28:32.960><c> some</c>\nmodels generation open QA like some\nmodels generation open QA like some question<01:28:33.480><c> answering</c><01:28:34.239><c> some</c><01:28:34.440><c> summarization</c>\nquestion answering some summarization\nquestion answering some summarization and<01:28:35.360><c> all</c><01:28:35.480><c> of</c><01:28:35.639><c> these</c><01:28:35.760><c> things</c><01:28:35.960><c> so</c><01:28:36.080><c> there's</c><01:28:36.239><c> so</c>\nand all of these things so there's so\nand all of these things so there's so many<01:28:36.520><c> things</c><01:28:36.679><c> you</c><01:28:36.800><c> have</c><01:28:36.920><c> to</c><01:28:37.360><c> cover</c><01:28:38.360><c> um</c><01:28:39.119><c> then</c>\nmany things you have to cover um then\nmany things you have to cover um then the<01:28:39.600><c> tasks</c><01:28:39.880><c> are</c><01:28:40.080><c> really</c><01:28:40.320><c> open-ended</c><01:28:41.080><c> so</c><01:28:41.239><c> it's</c>\nthe tasks are really open-ended so it's\nthe tasks are really open-ended so it's very<01:28:41.639><c> hard</c><01:28:41.840><c> to</c><01:28:42.000><c> automate</c><01:28:42.400><c> so</c><01:28:42.600><c> that's</c><01:28:42.880><c> what</c><01:28:43.000><c> you</c>\nvery hard to automate so that's what you\nvery hard to automate so that's what you were<01:28:43.400><c> alluding</c><01:28:43.760><c> to</c><01:28:44.440><c> before</c><01:28:45.440><c> so</c><01:28:45.679><c> the</c><01:28:45.840><c> idea</c><01:28:46.760><c> uh</c>\nwere alluding to before so the idea uh\nwere alluding to before so the idea uh is<01:28:47.159><c> that</c><01:28:47.360><c> instead</c><01:28:47.679><c> of</c><01:28:47.800><c> trying</c><01:28:48.080><c> to</c><01:28:48.239><c> come</c><01:28:48.400><c> up</c>\nis that instead of trying to come up\nis that instead of trying to come up with<01:28:49.000><c> really</c><01:28:49.280><c> easily</c><01:28:49.679><c> automated</c><01:28:50.560><c> uh</c>\nwith really easily automated uh\nwith really easily automated uh benchmarks<01:28:51.840><c> uh</c><01:28:52.040><c> it's</c><01:28:52.239><c> just</c><01:28:52.400><c> we're</c><01:28:52.520><c> going</c><01:28:52.639><c> to</c>\nbenchmarks uh it's just we're going to\nbenchmarks uh it's just we're going to ask<01:28:53.119><c> questions</c><01:28:53.560><c> that</c><01:28:53.880><c> that</c><01:28:54.360><c> users</c><01:28:54.760><c> actually</c>\nask questions that that users actually\nask questions that that users actually ask<01:28:55.199><c> to</c><01:28:55.360><c> these</c><01:28:55.520><c> models</c><01:28:55.920><c> in</c><01:28:56.119><c> practice</c><01:28:56.800><c> and</c>\nask to these models in practice and\nask to these models in practice and we're<01:28:57.080><c> just</c><01:28:57.199><c> going</c><01:28:57.320><c> to</c><01:28:57.520><c> ask</c><01:28:57.760><c> annotators</c><01:28:58.560><c> to</c>\nwe're just going to ask annotators to\nwe're just going to ask annotators to say<01:28:59.320><c> between</c><01:28:59.760><c> these</c><01:28:59.920><c> two</c><01:29:00.159><c> models</c><01:29:00.800><c> which</c><01:29:00.920><c> one</c>\nsay between these two models which one\nsay between these two models which one is<01:29:01.280><c> better</c><01:29:01.639><c> like</c><01:29:01.760><c> what's</c><01:29:01.960><c> the</c><01:29:02.239><c> what's</c><01:29:02.400><c> the</c>\nis better like what's the what's the\nis better like what's the what's the better<01:29:02.719><c> output</c><01:29:03.040><c> so</c><01:29:03.239><c> basically</c><01:29:03.600><c> do</c><01:29:03.800><c> exact</c><01:29:04.119><c> same</c>\nbetter output so basically do exact same\nbetter output so basically do exact same thing<01:29:05.239><c> as</c><01:29:06.159><c> um</c><01:29:07.119><c> basically</c><01:29:07.520><c> the</c><01:29:07.679><c> data</c><01:29:07.920><c> from</c><01:29:08.080><c> rhf</c>\nthing as um basically the data from rhf\nthing as um basically the data from rhf but<01:29:08.719><c> you</c><01:29:08.840><c> use</c><01:29:08.960><c> it</c><01:29:09.080><c> now</c><01:29:09.239><c> for</c><01:29:09.440><c> evaluation</c><01:29:10.320><c> yes</c>\nbut you use it now for evaluation yes\nbut you use it now for evaluation yes I'm<01:29:10.719><c> not</c><01:29:10.880><c> sure</c><01:29:11.040><c> I</c><01:29:11.400><c> understand</c><01:29:11.560><c> what</c><01:29:11.639><c> you</c><01:29:11.760><c> mean</c>\nI'm not sure I understand what you mean\nI'm not sure I understand what you mean by<01:29:12.199><c> like</c><01:29:12.400><c> can't</c><01:29:12.560><c> use</c><01:29:12.760><c> perplexity</c><01:29:13.239><c> and</c><01:29:13.360><c> not</c>\nby like can't use perplexity and not\nby like can't use perplexity and not calibrated<01:29:14.239><c> right</c><01:29:14.440><c> like</c><01:29:14.920><c> LM</c><01:29:15.360><c> is</c><01:29:15.520><c> still</c><01:29:15.800><c> doing</c>\ncalibrated right like LM is still doing\ncalibrated right like LM is still doing like<01:29:16.400><c> next</c><01:29:16.800><c> token</c>\nlike next token\nlike next token prediction<01:29:19.119><c> so</c><01:29:19.760><c> I</c><01:29:20.159><c> can't</c><01:29:21.159><c> so</c><01:29:21.400><c> think</c><01:29:21.600><c> about</c><01:29:22.440><c> um</c>\nprediction so I can't so think about um\nprediction so I can't so think about um the<01:29:23.639><c> optim</c><01:29:24.320><c> solution</c><01:29:24.719><c> after</c><01:29:24.920><c> doing</c><01:29:25.199><c> PO</c><01:29:25.800><c> is</c>\nthe optim solution after doing PO is\nthe optim solution after doing PO is basically<01:29:26.560><c> one</c><01:29:26.760><c> model</c><01:29:27.119><c> that</c><01:29:27.320><c> gives</c><01:29:27.560><c> you</c><01:29:28.119><c> uh</c>\nbasically one model that gives you uh\nbasically one model that gives you uh essentially<01:29:28.960><c> a</c><01:29:29.400><c> Delta</c><01:29:30.400><c> um</c><01:29:31.119><c> like</c><01:29:31.320><c> basically</c>\nessentially a Delta um like basically\nessentially a Delta um like basically says<01:29:31.880><c> that</c><01:29:32.040><c> there's</c><01:29:32.199><c> only</c><01:29:32.400><c> one</c><01:29:32.600><c> sentence</c><01:29:33.280><c> that</c>\nsays that there's only one sentence that\nsays that there's only one sentence that is<01:29:34.719><c> that</c><01:29:34.880><c> could</c><01:29:35.040><c> be</c><01:29:35.159><c> generated</c><01:29:35.800><c> for</c><01:29:36.040><c> that</c>\nis that could be generated for that\nis that could be generated for that question<01:29:36.960><c> so</c><01:29:37.159><c> now</c><01:29:37.360><c> if</c><01:29:37.440><c> you</c><01:29:37.600><c> use</c><01:29:37.760><c> it</c><01:29:37.920><c> on</c>\nquestion so now if you use it on\nquestion so now if you use it on something<01:29:38.360><c> that</c><01:29:38.440><c> is</c><01:29:38.600><c> slightly</c><01:29:39.000><c> semantically</c>\nsomething that is slightly semantically\nsomething that is slightly semantically differently<01:29:40.280><c> different</c><01:29:40.840><c> it</c><01:29:40.960><c> would</c><01:29:41.119><c> actually</c>\ndifferently different it would actually\ndifferently different it would actually give<01:29:41.440><c> a</c><01:29:41.560><c> likelihood</c><01:29:42.000><c> of</c><01:29:42.159><c> zero</c><01:29:42.719><c> for</c><01:29:42.960><c> that</c>\ngive a likelihood of zero for that\ngive a likelihood of zero for that answer<01:29:44.400><c> so</c><01:29:44.679><c> in</c><01:29:44.880><c> reality</c><01:29:45.239><c> it's</c><01:29:45.360><c> not</c><01:29:45.560><c> that</c>\nanswer so in reality it's not that\nanswer so in reality it's not that extreme<01:29:46.280><c> because</c><01:29:46.440><c> as</c><01:29:46.520><c> you</c><01:29:46.639><c> say</c><01:29:46.760><c> it's</c><01:29:46.880><c> still</c><01:29:47.040><c> a</c>\nextreme because as you say it's still a\nextreme because as you say it's still a distribution<01:29:47.560><c> but</c><01:29:47.679><c> I</c><01:29:47.800><c> just</c><01:29:48.080><c> shows</c><01:29:48.400><c> you</c><01:29:48.600><c> that</c>\ndistribution but I just shows you that\ndistribution but I just shows you that there's<01:29:48.880><c> a</c><01:29:49.080><c> there's</c><01:29:49.239><c> a</c><01:29:49.360><c> fundamental</c><01:29:49.880><c> issue</c>\nthere's a there's a fundamental issue\nthere's a there's a fundamental issue with<01:29:50.679><c> perplexity</c><01:29:51.600><c> once</c><01:29:51.920><c> these</c><01:29:52.119><c> models</c><01:29:52.760><c> are</c>\nwith perplexity once these models are\nwith perplexity once these models are not<01:29:54.040><c> llms</c><01:29:54.560><c> anymore</c><01:29:54.880><c> they</c><01:29:55.000><c> were</c><01:29:55.119><c> not</c><01:29:55.280><c> trained</c>\nnot llms anymore they were not trained\nnot llms anymore they were not trained at<01:29:56.159><c> least</c><01:29:56.320><c> with</c><01:29:56.440><c> P</c><01:29:56.800><c> they</c><01:29:56.880><c> were</c><01:29:57.000><c> not</c><01:29:57.119><c> trained</c><01:29:57.360><c> to</c>\nat least with P they were not trained to\nat least with P they were not trained to to<01:29:57.679><c> do</c><01:29:57.840><c> maximum</c><01:29:58.199><c> likelihood</c><01:29:58.639><c> anymore</c><01:29:59.119><c> they</c>\nto do maximum likelihood anymore they\nto do maximum likelihood anymore they were<01:29:59.360><c> trained</c><01:29:59.600><c> to</c><01:29:59.679><c> be</c>\npolicies<01:30:03.360><c> okay</c><01:30:04.040><c> um</c><01:30:04.400><c> so</c><01:30:04.840><c> probably</c><01:30:05.159><c> the</c><01:30:05.280><c> most</c>\npolicies okay um so probably the most\npolicies okay um so probably the most common<01:30:06.000><c> or</c><01:30:06.199><c> like</c><01:30:06.400><c> the</c><01:30:06.560><c> most</c><01:30:07.520><c> um</c><01:30:08.520><c> yeah</c><01:30:08.679><c> the</c><01:30:08.760><c> most</c>\ncommon or like the most um yeah the most\ncommon or like the most um yeah the most common<01:30:09.320><c> Benchmark</c><01:30:09.800><c> or</c><01:30:10.080><c> the</c><01:30:10.159><c> most</c><01:30:10.320><c> trusted</c><01:30:10.719><c> one</c>\ncommon Benchmark or the most trusted one\ncommon Benchmark or the most trusted one is<01:30:11.159><c> what</c><01:30:11.280><c> we</c><01:30:11.400><c> call</c><01:30:11.600><c> Chad</c><01:30:11.960><c> uh</c><01:30:12.080><c> sorry</c><01:30:12.280><c> chatbot</c>\nis what we call Chad uh sorry chatbot\nis what we call Chad uh sorry chatbot Arena<01:30:13.639><c> uh</c><01:30:13.760><c> which</c><01:30:13.880><c> is</c><01:30:14.119><c> basically</c><01:30:14.800><c> go</c><01:30:15.000><c> on</c>\nArena uh which is basically go on\nArena uh which is basically go on internet<01:30:15.760><c> have</c><01:30:16.000><c> random</c><01:30:16.320><c> users</c><01:30:16.679><c> on</c><01:30:16.840><c> the</c>\ninternet have random users on the\ninternet have random users on the internet<01:30:17.600><c> blindly</c><01:30:18.520><c> talk</c><01:30:18.800><c> with</c><01:30:19.000><c> two</c><01:30:19.199><c> chat</c><01:30:19.440><c> Bots</c>\ninternet blindly talk with two chat Bots\ninternet blindly talk with two chat Bots just<01:30:20.199><c> ask</c><01:30:20.520><c> many</c><01:30:20.800><c> questions</c><01:30:21.320><c> see</c><01:30:21.600><c> the</c><01:30:21.719><c> two</c>\njust ask many questions see the two\njust ask many questions see the two answers<01:30:22.480><c> and</c><01:30:22.679><c> rate</c><01:30:22.960><c> which</c><01:30:23.080><c> one</c><01:30:23.199><c> is</c><01:30:23.400><c> better</c><01:30:23.800><c> and</c>\nanswers and rate which one is better and\nanswers and rate which one is better and and<01:30:24.080><c> you</c><01:30:24.159><c> do</c><01:30:24.320><c> that</c><01:30:24.480><c> over</c><01:30:25.159><c> hundred</c><01:30:25.440><c> of</c>\nand you do that over hundred of\nand you do that over hundred of thousands<01:30:25.880><c> of</c><01:30:26.040><c> users</c><01:30:26.639><c> and</c><01:30:26.760><c> then</c><01:30:26.880><c> you</c><01:30:27.080><c> get</c><01:30:27.760><c> uh</c>\nthousands of users and then you get uh\nthousands of users and then you get uh the<01:30:28.080><c> actual</c><01:30:28.360><c> preferences</c><01:30:28.920><c> and</c><01:30:29.040><c> you</c><01:30:29.199><c> get</c>\nthe actual preferences and you get\nthe actual preferences and you get rankings<01:30:29.920><c> of</c><01:30:30.080><c> models</c><01:30:30.960><c> uh</c><01:30:31.040><c> so</c><01:30:31.239><c> you</c><01:30:31.320><c> can</c><01:30:31.520><c> go</c>\nrankings of models uh so you can go\nrankings of models uh so you can go right<01:30:31.960><c> now</c><01:30:32.440><c> on</c><01:30:32.679><c> chatbot</c><01:30:33.119><c> Arena</c><01:30:33.520><c> and</c><01:30:33.679><c> actually</c>\nright now on chatbot Arena and actually\nright now on chatbot Arena and actually interact<01:30:34.440><c> with</c><01:30:34.600><c> these</c><01:30:34.719><c> models</c><01:30:35.679><c> um</c><01:30:36.199><c> one</c>\ninteract with these models um one\ninteract with these models um one potential<01:30:37.000><c> issue</c><01:30:37.480><c> just</c><01:30:37.639><c> to</c><01:30:37.880><c> highlight</c><01:30:38.560><c> is</c>\npotential issue just to highlight is\npotential issue just to highlight is that<01:30:38.920><c> while</c><01:30:39.119><c> people</c><01:30:39.360><c> who</c><01:30:39.560><c> want</c><01:30:39.679><c> to</c><01:30:39.840><c> do</c><01:30:40.040><c> these</c>\nthat while people who want to do these\nthat while people who want to do these type<01:30:40.360><c> of</c><01:30:40.480><c> things</c><01:30:40.600><c> are</c><01:30:40.760><c> usually</c><01:30:41.000><c> more</c><01:30:41.199><c> like</c>\ntype of things are usually more like\ntype of things are usually more like Tech<01:30:41.560><c> driven</c><01:30:42.400><c> um</c><01:30:42.560><c> or</c><01:30:42.760><c> like</c><01:30:43.159><c> techsavvy</c><01:30:43.679><c> uh</c><01:30:44.400><c> so</c><01:30:44.639><c> a</c>\nTech driven um or like techsavvy uh so a\nTech driven um or like techsavvy uh so a lot<01:30:44.840><c> of</c><01:30:44.960><c> the</c><01:30:45.040><c> questions</c><01:30:45.320><c> that</c><01:30:45.440><c> you</c><01:30:45.520><c> will</c><01:30:45.679><c> ask</c>\nlot of the questions that you will ask\nlot of the questions that you will ask are<01:30:46.239><c> more</c><01:30:46.440><c> like</c><01:30:46.639><c> Tech</c><01:30:46.880><c> stuff</c><01:30:47.400><c> discussing</c>\nare more like Tech stuff discussing\nare more like Tech stuff discussing software<01:30:48.320><c> errors</c><01:30:48.960><c> inquiries</c><01:30:49.440><c> about</c><01:30:49.679><c> AI</c><01:30:49.960><c> tools</c>\nsoftware errors inquiries about AI tools\nsoftware errors inquiries about AI tools and<01:30:50.480><c> all</c><01:30:50.719><c> these</c><01:30:50.880><c> things</c><01:30:51.960><c> um</c><01:30:52.960><c> so</c><01:30:53.320><c> another</c><01:30:53.560><c> issue</c>\nand all these things um so another issue\nand all these things um so another issue is<01:30:54.119><c> cost</c><01:30:54.320><c> and</c><01:30:54.480><c> speed</c><01:30:54.840><c> if</c><01:30:54.960><c> you</c><01:30:55.080><c> really</c><01:30:55.239><c> want</c><01:30:55.360><c> to</c>\nis cost and speed if you really want to\nis cost and speed if you really want to use<01:30:55.719><c> something</c><01:30:56.000><c> like</c><01:30:56.199><c> this</c><01:30:56.480><c> for</c><01:30:56.760><c> development</c>\nuse something like this for development\nuse something like this for development process<01:30:58.360><c> um</c><01:30:58.719><c> it</c><01:30:58.840><c> will</c><01:30:58.960><c> be</c><01:30:59.119><c> too</c><01:30:59.360><c> costly</c><01:30:59.840><c> because</c>\nprocess um it will be too costly because\nprocess um it will be too costly because you<01:31:00.080><c> would</c><01:31:00.239><c> need</c><01:31:00.400><c> to</c><01:31:00.960><c> basically</c><01:31:01.320><c> pay</c><01:31:01.480><c> a</c><01:31:01.560><c> lot</c><01:31:01.639><c> of</c>\nyou would need to basically pay a lot of\nyou would need to basically pay a lot of humans<01:31:02.040><c> to</c><01:31:02.199><c> do</c><01:31:02.920><c> that</c><01:31:03.920><c> so</c><01:31:04.159><c> one</c><01:31:04.400><c> simple</c><01:31:04.760><c> idea</c><01:31:05.600><c> is</c>\nhumans to do that so one simple idea is\nhumans to do that so one simple idea is again<01:31:06.480><c> as</c><01:31:06.719><c> we</c><01:31:06.960><c> said</c><01:31:07.199><c> many</c><01:31:07.480><c> times</c><01:31:07.920><c> just</c><01:31:08.080><c> use</c><01:31:08.320><c> LM</c>\nagain as we said many times just use LM\nagain as we said many times just use LM instead<01:31:09.000><c> of</c><01:31:09.199><c> humans</c><01:31:10.199><c> uh</c><01:31:10.440><c> you</c><01:31:10.639><c> probably</c><01:31:10.880><c> know</c>\ninstead of humans uh you probably know\ninstead of humans uh you probably know the<01:31:11.199><c> drill</c><01:31:11.719><c> at</c><01:31:11.880><c> this</c><01:31:12.040><c> point</c><01:31:12.920><c> uh</c><01:31:13.080><c> steps</c><01:31:13.520><c> for</c>\nthe drill at this point uh steps for\nthe drill at this point uh steps for every<01:31:14.119><c> instruction</c><01:31:14.639><c> generate</c><01:31:15.199><c> outputs</c><01:31:15.760><c> by</c>\nevery instruction generate outputs by\nevery instruction generate outputs by some<01:31:16.199><c> baseline</c><01:31:17.000><c> and</c><01:31:17.159><c> the</c><01:31:17.320><c> model</c><01:31:17.600><c> that</c><01:31:17.679><c> you</c>\nsome baseline and the model that you\nsome baseline and the model that you want<01:31:17.880><c> to</c><01:31:18.280><c> evaluate</c><01:31:19.280><c> um</c><01:31:19.480><c> so</c><01:31:19.719><c> here</c><01:31:19.880><c> you</c><01:31:20.040><c> imagine</c>\nwant to evaluate um so here you imagine\nwant to evaluate um so here you imagine that<01:31:20.679><c> I</c><01:31:20.960><c> I'm</c><01:31:21.199><c> comparing</c><01:31:21.880><c> an</c><01:31:22.080><c> answer</c><01:31:22.440><c> from</c><01:31:22.600><c> Chad</c>\nthat I I'm comparing an answer from Chad\nthat I I'm comparing an answer from Chad GPT<01:31:23.440><c> and</c><01:31:23.560><c> from</c>\nGPT and from\nGPT and from I'm<01:31:24.840><c> just</c><01:31:25.080><c> asking</c><01:31:25.400><c> a</c><01:31:25.560><c> model</c><01:31:26.520><c> uh</c><01:31:26.719><c> another</c><01:31:27.119><c> model</c>\nI'm just asking a model uh another model\nI'm just asking a model uh another model uh<01:31:28.159><c> which</c><01:31:28.320><c> one</c><01:31:28.480><c> is</c><01:31:28.679><c> better</c><01:31:29.520><c> and</c><01:31:29.800><c> I</c><01:31:30.000><c> just</c>\nuh which one is better and I just\nuh which one is better and I just basically<01:31:31.040><c> average</c><01:31:31.440><c> that</c><01:31:31.600><c> out</c><01:31:32.320><c> uh</c><01:31:32.480><c> yeah</c><01:31:32.600><c> I</c>\nbasically average that out uh yeah I\nbasically average that out uh yeah I asked<01:31:33.159><c> gp4</c><01:31:33.679><c> which</c><01:31:33.800><c> one</c><01:31:33.920><c> is</c><01:31:34.040><c> better</c><01:31:34.520><c> I</c><01:31:34.639><c> average</c>\nasked gp4 which one is better I average\nasked gp4 which one is better I average that<01:31:35.159><c> out</c><01:31:35.320><c> over</c><01:31:35.639><c> my</c><01:31:35.760><c> entire</c><01:31:36.199><c> distribution</c>\nthat out over my entire distribution\nthat out over my entire distribution over<01:31:37.400><c> my</c><01:31:37.600><c> entire</c><01:31:37.920><c> Benchmark</c><01:31:38.360><c> or</c><01:31:38.560><c> data</c><01:31:38.800><c> set</c><01:31:39.280><c> and</c>\nover my entire Benchmark or data set and\nover my entire Benchmark or data set and that<01:31:39.719><c> gives</c><01:31:39.960><c> me</c><01:31:40.159><c> a</c><01:31:40.560><c> RN</c><01:31:40.840><c> rate</c><01:31:41.080><c> so</c><01:31:41.360><c> RN</c>\nthat gives me a RN rate so RN\nthat gives me a RN rate so RN probability<01:31:42.440><c> for</c><01:31:42.679><c> one</c><01:31:42.880><c> model</c><01:31:43.520><c> compared</c><01:31:43.800><c> to</c>\nprobability for one model compared to\nprobability for one model compared to another<01:31:44.159><c> one</c><01:31:44.600><c> and</c><01:31:44.840><c> now</c><01:31:44.960><c> you</c><01:31:45.040><c> can</c><01:31:45.199><c> rank</c><01:31:45.719><c> models</c>\nanother one and now you can rank models\nanother one and now you can rank models uh<01:31:46.840><c> and</c><01:31:46.960><c> this</c><01:31:47.080><c> is</c><01:31:47.199><c> the</c><01:31:47.320><c> Alpa</c><01:31:47.760><c> eval</c><01:31:48.520><c> uh</c>\nuh and this is the Alpa eval uh\nuh and this is the Alpa eval uh leaderboard<01:31:50.520><c> so</c><01:31:50.880><c> the</c><01:31:51.000><c> benefits</c><01:31:51.400><c> of</c><01:31:51.639><c> this</c><01:31:52.000><c> is</c>\nleaderboard so the benefits of this is\nleaderboard so the benefits of this is that<01:31:52.440><c> actually</c><01:31:52.800><c> we</c><01:31:52.960><c> show</c><01:31:53.400><c> we</c><01:31:53.560><c> get</c><01:31:53.920><c> 98%</c>\nthat actually we show we get 98%\nthat actually we show we get 98% correlation<01:31:55.000><c> with</c><01:31:55.119><c> Chad</c><01:31:55.360><c> B</c><01:31:55.520><c> Arena</c><01:31:55.920><c> so</c><01:31:56.159><c> very</c>\ncorrelation with Chad B Arena so very\ncorrelation with Chad B Arena so very high<01:31:56.560><c> correlation</c><01:31:57.040><c> with</c><01:31:57.280><c> humans</c><01:31:58.360><c> um</c><01:31:59.360><c> so</c><01:31:59.639><c> this</c>\nhigh correlation with humans um so this\nhigh correlation with humans um so this is<01:32:00.360><c> yeah</c><01:32:00.679><c> comparison</c><01:32:01.159><c> with</c><01:32:01.239><c> correlation</c><01:32:01.639><c> with</c>\nis yeah comparison with correlation with\nis yeah comparison with correlation with other<01:32:01.960><c> benchmarks</c><01:32:02.920><c> and</c><01:32:03.040><c> it</c><01:32:03.199><c> takes</c><01:32:03.440><c> less</c><01:32:03.600><c> than</c>\nother benchmarks and it takes less than\nother benchmarks and it takes less than three<01:32:03.920><c> minutes</c><01:32:04.199><c> and</c><01:32:04.400><c> less</c><01:32:04.560><c> than</c><01:32:04.639><c> $10</c><01:32:05.199><c> to</c><01:32:05.320><c> run</c>\nthree minutes and less than $10 to run\nthree minutes and less than $10 to run so<01:32:05.639><c> it's</c><01:32:05.760><c> pretty</c><01:32:06.040><c> cheap</c><01:32:07.040><c> um</c><01:32:07.280><c> there</c><01:32:07.400><c> are</c>\nso it's pretty cheap um there are\nso it's pretty cheap um there are downsides<01:32:08.199><c> though</c><01:32:08.760><c> uh</c><01:32:08.880><c> one</c><01:32:09.000><c> of</c><01:32:09.159><c> them</c><01:32:09.400><c> is</c><01:32:09.600><c> purus</c>\ndownsides though uh one of them is purus\ndownsides though uh one of them is purus correlation<01:32:11.320><c> um</c><01:32:11.719><c> so</c><01:32:12.119><c> as</c><01:32:12.239><c> we</c><01:32:12.400><c> already</c><01:32:12.639><c> saw</c>\ncorrelation um so as we already saw\ncorrelation um so as we already saw before<01:32:13.960><c> LMS</c><01:32:14.520><c> prefer</c><01:32:15.040><c> this</c><01:32:15.119><c> is</c><01:32:15.239><c> one</c><01:32:15.400><c> SP</c>\nbefore LMS prefer this is one SP\nbefore LMS prefer this is one SP correlation<01:32:16.119><c> not</c><01:32:16.280><c> many</c><01:32:16.520><c> I'll</c><01:32:16.639><c> just</c><01:32:16.960><c> talk</c>\ncorrelation not many I'll just talk\ncorrelation not many I'll just talk about<01:32:17.320><c> one</c><01:32:17.560><c> LMS</c><01:32:18.000><c> prefer</c><01:32:18.239><c> longer</c><01:32:18.520><c> outputs</c>\nabout one LMS prefer longer outputs\nabout one LMS prefer longer outputs actually<01:32:19.199><c> humans</c><01:32:19.520><c> also</c><01:32:19.719><c> prefer</c><01:32:20.000><c> longer</c>\nactually humans also prefer longer\nactually humans also prefer longer outputs<01:32:21.000><c> but</c><01:32:21.119><c> the</c><01:32:21.320><c> problem</c><01:32:21.840><c> or</c><01:32:22.040><c> the</c><01:32:22.199><c> issue</c>\noutputs but the problem or the issue\noutputs but the problem or the issue once<01:32:22.600><c> you</c><01:32:22.679><c> use</c><01:32:22.840><c> llms</c><01:32:23.320><c> is</c><01:32:23.400><c> that</c><01:32:23.520><c> once</c><01:32:23.679><c> there</c>\nonce you use llms is that once there\nonce you use llms is that once there bias<01:32:24.560><c> you</c><01:32:24.679><c> will</c><01:32:24.880><c> continue</c><01:32:25.239><c> optimizing</c><01:32:25.800><c> that</c>\nbias you will continue optimizing that\nbias you will continue optimizing that humans<01:32:26.719><c> at</c><01:32:26.840><c> some</c><01:32:27.000><c> point</c><01:32:27.360><c> I</c><01:32:27.440><c> can</c><01:32:27.600><c> guarantee</c><01:32:27.960><c> you</c>\nhumans at some point I can guarantee you\nhumans at some point I can guarantee you if<01:32:28.159><c> I</c><01:32:28.280><c> ask</c><01:32:28.440><c> a</c><01:32:28.560><c> simple</c><01:32:28.840><c> question</c><01:32:29.080><c> and</c><01:32:29.199><c> you</c><01:32:29.320><c> give</c>\nif I ask a simple question and you give\nif I ask a simple question and you give me<01:32:29.719><c> five</c><01:32:29.960><c> pages</c><01:32:30.239><c> of</c><01:32:30.440><c> answers</c><01:32:30.960><c> I'll</c><01:32:31.119><c> be</c><01:32:31.239><c> like</c><01:32:31.360><c> no</c>\nme five pages of answers I'll be like no\nme five pages of answers I'll be like no I<01:32:31.520><c> don't</c><01:32:31.679><c> like</c><01:32:31.800><c> that</c><01:32:32.000><c> answer</c><01:32:32.520><c> but</c><01:32:32.679><c> LMS</c><01:32:33.119><c> if</c><01:32:33.239><c> they</c>\nI don't like that answer but LMS if they\nI don't like that answer but LMS if they have<01:32:33.560><c> this</c><01:32:33.719><c> bius</c><01:32:34.040><c> and</c><01:32:34.119><c> they</c><01:32:34.199><c> were</c><01:32:34.320><c> trained</c><01:32:34.600><c> for</c>\nhave this bius and they were trained for\nhave this bius and they were trained for that<01:32:35.159><c> they</c><01:32:35.239><c> will</c><01:32:35.400><c> continue</c><01:32:35.760><c> preferring</c>\nthat they will continue preferring\nthat they will continue preferring longer<01:32:36.840><c> outputs</c><01:32:37.840><c> so</c><01:32:38.800><c> uh</c><01:32:38.960><c> here</c><01:32:39.159><c> we</c><01:32:39.320><c> see</c><01:32:40.159><c> um</c><01:32:41.159><c> the</c>\nlonger outputs so uh here we see um the\nlonger outputs so uh here we see um the the<01:32:41.600><c> preference</c><01:32:42.320><c> just</c><01:32:42.440><c> showing</c><01:32:42.800><c> that</c><01:32:42.960><c> like</c>\nthe preference just showing that like\nthe preference just showing that like humans<01:32:43.480><c> and</c><01:32:43.679><c> models</c><01:32:44.080><c> prefer</c><01:32:44.360><c> longer</c><01:32:44.960><c> outputs</c>\nhumans and models prefer longer outputs\nhumans and models prefer longer outputs um<01:32:46.480><c> and</c><01:32:46.719><c> here</c><01:32:46.840><c> is</c><01:32:47.080><c> another</c><01:32:47.639><c> view</c><01:32:48.199><c> of</c><01:32:48.360><c> the</c>\num and here is another view of the\num and here is another view of the initial<01:32:48.800><c> apaka</c><01:32:49.199><c> eval</c><01:32:49.520><c> data</c><01:32:50.159><c> uh</c><01:32:50.360><c> Benchmark</c>\ninitial apaka eval data uh Benchmark\ninitial apaka eval data uh Benchmark where<01:32:51.520><c> when</c><01:32:51.679><c> we</c><01:32:51.960><c> asked</c><01:32:52.960><c> um</c><01:32:53.520><c> when</c><01:32:53.639><c> we</c><01:32:53.840><c> we</c><01:32:53.960><c> rank</c>\nwhere when we asked um when we we rank\nwhere when we asked um when we we rank gp4<01:32:55.159><c> when</c><01:32:55.320><c> we</c><01:32:55.440><c> look</c><01:32:55.560><c> at</c><01:32:55.679><c> the</c><01:32:55.800><c> Run</c><01:32:56.000><c> rate</c><01:32:56.199><c> of</c><01:32:56.320><c> gp4</c>\ngp4 when we look at the Run rate of gp4\ngp4 when we look at the Run rate of gp4 versus<01:32:57.679><c> actually</c><01:32:58.320><c> uh</c><01:32:58.440><c> gp4</c><01:32:59.119><c> itself</c><01:32:59.719><c> if</c><01:32:59.880><c> we</c><01:33:00.080><c> com</c>\nversus actually uh gp4 itself if we com\nversus actually uh gp4 itself if we com if<01:33:00.400><c> we</c><01:33:00.480><c> use</c><01:33:00.639><c> the</c><01:33:00.760><c> standard</c><01:33:01.080><c> GPT</c><01:33:01.360><c> 4</c><01:33:01.520><c> it</c><01:33:01.600><c> gets</c><01:33:01.840><c> 50%</c>\nif we use the standard GPT 4 it gets 50%\nif we use the standard GPT 4 it gets 50% kind<01:33:02.719><c> of</c><01:33:02.840><c> by</c><01:33:03.000><c> definition</c><01:33:03.440><c> because</c><01:33:03.600><c> we're</c>\nkind of by definition because we're\nkind of by definition because we're comparing<01:33:04.280><c> GPT</c><01:33:04.719><c> 4</c><01:33:05.000><c> versus</c><01:33:05.280><c> gp4</c><01:33:06.080><c> but</c><01:33:06.239><c> if</c><01:33:06.400><c> we</c><01:33:06.679><c> ask</c>\ncomparing GPT 4 versus gp4 but if we ask\ncomparing GPT 4 versus gp4 but if we ask a<01:33:07.440><c> gbd4</c><01:33:08.000><c> to</c><01:33:08.080><c> be</c><01:33:08.199><c> slightly</c><01:33:08.520><c> more</c><01:33:08.679><c> verose</c><01:33:09.159><c> so</c><01:33:09.320><c> we</c>\na gbd4 to be slightly more verose so we\na gbd4 to be slightly more verose so we just<01:33:09.600><c> say</c><01:33:09.920><c> in</c><01:33:10.000><c> the</c><01:33:10.159><c> prompt</c><01:33:10.600><c> be</c><01:33:10.880><c> Vos</c><01:33:11.280><c> in</c><01:33:11.360><c> your</c>\njust say in the prompt be Vos in your\njust say in the prompt be Vos in your answers<01:33:12.199><c> then</c><01:33:12.320><c> it</c><01:33:12.400><c> gets</c><01:33:12.560><c> a</c><01:33:12.679><c> r</c><01:33:12.880><c> rate</c><01:33:13.080><c> of</c>\nanswers then it gets a r rate of\nanswers then it gets a r rate of 64.4%<01:33:14.880><c> so</c><01:33:15.159><c> really</c><01:33:15.600><c> there's</c><01:33:15.760><c> a</c><01:33:15.960><c> huge</c><01:33:16.239><c> variance</c>\n64.4% so really there's a huge variance\n64.4% so really there's a huge variance and<01:33:16.719><c> if</c><01:33:16.800><c> we</c><01:33:16.920><c> ask</c><01:33:17.040><c> it</c><01:33:17.159><c> to</c><01:33:17.239><c> be</c><01:33:17.360><c> concise</c><01:33:17.719><c> it</c><01:33:17.840><c> gets</c>\nand if we ask it to be concise it gets\nand if we ask it to be concise it gets 20%<01:33:18.800><c> so</c><01:33:18.960><c> there's</c><01:33:19.119><c> a</c><01:33:19.280><c> huge</c><01:33:19.560><c> variance</c><01:33:20.199><c> depending</c>\n20% so there's a huge variance depending\n20% so there's a huge variance depending on<01:33:21.480><c> um</c><01:33:22.280><c> whether</c><01:33:22.480><c> you</c><01:33:22.639><c> ask</c><01:33:22.800><c> it</c><01:33:22.880><c> to</c><01:33:22.960><c> be</c><01:33:23.080><c> concise</c>\non um whether you ask it to be concise\non um whether you ask it to be concise of\nof\nof that's<01:33:24.480><c> very</c><01:33:24.800><c> annoying</c><01:33:25.800><c> um</c><01:33:26.159><c> so</c><01:33:26.840><c> one</c><01:33:27.119><c> possible</c>\nthat's very annoying um so one possible\nthat's very annoying um so one possible solution<01:33:28.119><c> which</c><01:33:28.280><c> is</c><01:33:28.440><c> what</c><01:33:28.600><c> we</c><01:33:28.800><c> did</c><01:33:29.320><c> is</c><01:33:29.679><c> uh</c><01:33:29.800><c> just</c>\nsolution which is what we did is uh just\nsolution which is what we did is uh just use<01:33:30.159><c> some</c><01:33:30.320><c> regression</c><01:33:30.840><c> analysis</c><01:33:31.480><c> I'm</c><01:33:31.560><c> not</c>\nuse some regression analysis I'm not\nuse some regression analysis I'm not going<01:33:31.840><c> to</c><01:33:32.000><c> go</c><01:33:32.159><c> into</c><01:33:32.400><c> details</c><01:33:32.760><c> but</c><01:33:32.920><c> basically</c>\ngoing to go into details but basically\ngoing to go into details but basically use<01:33:33.520><c> Cal</c><01:33:33.840><c> inference</c><01:33:34.199><c> tools</c><01:33:34.719><c> to</c><01:33:34.880><c> control</c><01:33:35.199><c> for</c>\nuse Cal inference tools to control for\nuse Cal inference tools to control for length<01:33:36.080><c> and</c><01:33:36.320><c> right</c><01:33:36.560><c> now</c><01:33:37.199><c> uh</c><01:33:37.440><c> actually</c><01:33:37.679><c> length</c>\nlength and right now uh actually length\nlength and right now uh actually length matters<01:33:38.320><c> much</c><01:33:38.520><c> less</c><01:33:38.800><c> so</c><01:33:39.239><c> if</c><01:33:39.360><c> you</c><01:33:39.480><c> ask</c><01:33:39.639><c> it</c><01:33:39.760><c> to</c><01:33:39.840><c> be</c>\nmatters much less so if you ask it to be\nmatters much less so if you ask it to be veros<01:33:40.360><c> we</c><01:33:40.480><c> still</c><01:33:40.679><c> get</c><01:33:40.840><c> some</c><01:33:41.080><c> gains</c><01:33:41.679><c> but</c><01:33:41.920><c> much</c>\nveros we still get some gains but much\nveros we still get some gains but much less<01:33:44.600><c> great</c><01:33:45.000><c> so</c><01:33:45.239><c> that's</c><01:33:45.480><c> all</c><01:33:45.679><c> about</c><01:33:45.920><c> post</c>\nless great so that's all about post\nless great so that's all about post training<01:33:46.760><c> and</c><01:33:46.960><c> now</c><01:33:47.159><c> for</c><01:33:47.360><c> the</c><01:33:47.520><c> next</c><01:33:47.920><c> eight</c>\ntraining and now for the next eight\ntraining and now for the next eight minutes<01:33:48.840><c> I</c><01:33:49.040><c> might</c><01:33:49.239><c> talk</c><01:33:49.440><c> about</c><01:33:49.719><c> systems</c><01:33:50.199><c> or</c>\nminutes I might talk about systems or\nminutes I might talk about systems or just<01:33:50.560><c> answer</c><01:33:50.920><c> questions</c><01:33:51.480><c> yes</c><01:33:52.239><c> can</c><01:33:52.440><c> you</c><01:33:52.840><c> um</c><01:33:54.080><c> go</c>\njust answer questions yes can you um go\njust answer questions yes can you um go back<01:33:54.440><c> to</c><01:33:54.600><c> your</c><01:33:55.080><c> post</c><01:33:55.360><c> training</c><01:33:55.639><c> in</c><01:33:55.760><c> terms</c><01:33:55.960><c> of</c>\nback to your post training in terms of\nback to your post training in terms of post<01:33:56.719><c> training</c><01:33:57.719><c> how</c><01:33:57.920><c> did</c><01:33:58.119><c> we</c><01:33:58.320><c> tune</c><01:33:58.679><c> those</c>\npost training how did we tune those\npost training how did we tune those parameters<01:33:59.520><c> using</c><01:33:59.920><c> the</c><01:34:00.520><c> small</c><01:34:01.119><c> body</c><01:34:01.480><c> of</c>\nparameters using the small body of\nparameters using the small body of fine-tuning<01:34:02.960><c> data</c><01:34:03.400><c> and</c><01:34:03.600><c> have</c><01:34:03.800><c> such</c><01:34:04.080><c> big</c>\nfine-tuning data and have such big\nfine-tuning data and have such big effect<01:34:04.639><c> on</c><01:34:04.760><c> the</c><01:34:04.920><c> model</c><01:34:05.440><c> you</c><01:34:05.560><c> mentioned</c>\neffect on the model you mentioned\neffect on the model you mentioned earlier<01:34:06.320><c> that</c><01:34:06.639><c> there's</c><01:34:06.840><c> a</c><01:34:07.000><c> different</c><01:34:07.239><c> set</c><01:34:07.440><c> of</c>\nearlier that there's a different set of\nearlier that there's a different set of hyperparameters<01:34:08.840><c> are</c><01:34:09.000><c> we</c><01:34:09.119><c> changing</c><01:34:09.800><c> just</c>\nhyperparameters are we changing just\nhyperparameters are we changing just some<01:34:10.199><c> of</c><01:34:10.360><c> the</c><01:34:10.480><c> weights</c><01:34:10.800><c> the</c><01:34:10.920><c> later</c><01:34:11.280><c> weights</c><01:34:11.600><c> or</c>\nsome of the weights the later weights or\nsome of the weights the later weights or all<01:34:12.080><c> the</c><01:34:12.239><c> weights</c><01:34:12.679><c> what's</c><01:34:12.920><c> actually</c>\nall the weights what's actually\nall the weights what's actually happening<01:34:14.080><c> yeah</c><01:34:14.679><c> uh</c><01:34:14.880><c> yeah</c><01:34:15.040><c> I</c><01:34:15.199><c> I</c><01:34:15.280><c> kind</c><01:34:15.400><c> of</c>\nhappening yeah uh yeah I I kind of\nhappening yeah uh yeah I I kind of skimmed<01:34:15.840><c> through</c><01:34:16.000><c> all</c><01:34:16.080><c> of</c><01:34:16.239><c> this</c><01:34:16.560><c> you</c><01:34:16.719><c> change</c>\nskimmed through all of this you change\nskimmed through all of this you change all<01:34:17.119><c> the</c><01:34:17.239><c> weights</c><01:34:17.920><c> actually</c><01:34:18.560><c> um</c><01:34:19.000><c> industry</c>\nall the weights actually um industry\nall the weights actually um industry would<01:34:19.639><c> change</c><01:34:19.880><c> all</c><01:34:20.000><c> the</c><01:34:20.159><c> weights</c><01:34:20.679><c> in</c><01:34:20.920><c> open</c>\nwould change all the weights in open\nwould change all the weights in open source<01:34:21.520><c> land</c><01:34:22.000><c> you</c><01:34:22.119><c> might</c><01:34:22.360><c> have</c><01:34:22.639><c> heard</c><01:34:22.920><c> of</c>\nsource land you might have heard of\nsource land you might have heard of Laura<01:34:23.920><c> which</c><01:34:24.080><c> is</c><01:34:24.239><c> going</c><01:34:24.360><c> to</c><01:34:24.880><c> change</c><01:34:25.880><c> basically</c>\nLaura which is going to change basically\nLaura which is going to change basically only<01:34:26.679><c> some</c><01:34:26.960><c> of</c><01:34:27.080><c> the</c><01:34:27.199><c> weights</c><01:34:27.719><c> or</c><01:34:27.920><c> it</c><01:34:28.080><c> actually</c>\nonly some of the weights or it actually\nonly some of the weights or it actually to<01:34:28.600><c> be</c><01:34:28.760><c> more</c><01:34:29.000><c> specific</c><01:34:29.639><c> it's</c><01:34:29.800><c> going</c><01:34:29.880><c> to</c><01:34:30.159><c> add</c>\nto be more specific it's going to add\nto be more specific it's going to add some<01:34:30.719><c> differences</c><01:34:31.159><c> to</c><01:34:31.320><c> the</c><01:34:31.440><c> output</c><01:34:31.840><c> of</c><01:34:32.040><c> every</c>\nsome differences to the output of every\nsome differences to the output of every of<01:34:32.560><c> every</c><01:34:32.760><c> layer</c><01:34:33.360><c> but</c><01:34:33.560><c> but</c><01:34:33.639><c> in</c><01:34:33.800><c> Industry</c>\nof every layer but but in Industry\nof every layer but but in Industry you're<01:34:34.239><c> going</c><01:34:34.360><c> to</c><01:34:34.520><c> just</c><01:34:34.719><c> fine</c><01:34:34.960><c> tune</c><01:34:35.199><c> all</c><01:34:35.400><c> the</c>\nyou're going to just fine tune all the\nyou're going to just fine tune all the weights<01:34:37.000><c> um</c><01:34:37.840><c> and</c><01:34:38.840><c> also</c><01:34:39.080><c> to</c><01:34:39.199><c> say</c><01:34:39.360><c> something</c>\nweights um and also to say something\nweights um and also to say something else<01:34:39.800><c> about</c><01:34:39.960><c> the</c><01:34:40.119><c> data</c><01:34:40.480><c> actually</c><01:34:40.719><c> the</c><01:34:40.880><c> SL</c><01:34:41.239><c> St</c>\nelse about the data actually the SL St\nelse about the data actually the SL St all<01:34:41.600><c> HF</c><01:34:42.119><c> you</c><01:34:42.280><c> usually</c><01:34:42.560><c> going</c><01:34:42.679><c> to</c><01:34:42.800><c> collect</c><01:34:43.800><c> uh</c><01:34:44.040><c> a</c>\nall HF you usually going to collect uh a\nall HF you usually going to collect uh a lot<01:34:44.400><c> more</c><01:34:44.600><c> data</c><01:34:44.880><c> than</c><01:34:45.040><c> with</c><01:34:45.199><c> sft</c><01:34:45.679><c> so</c><01:34:45.840><c> if</c><01:34:46.000><c> fft</c><01:34:46.600><c> is</c>\nlot more data than with sft so if fft is\nlot more data than with sft so if fft is like<01:34:47.159><c> 5,000</c><01:34:48.159><c> 10,000</c><01:34:48.840><c> maybe</c><01:34:49.080><c> 50,000</c><01:34:50.080><c> with</c><01:34:50.360><c> rhf</c>\nlike 5,000 10,000 maybe 50,000 with rhf\nlike 5,000 10,000 maybe 50,000 with rhf I<01:34:51.320><c> think</c><01:34:51.480><c> you're</c><01:34:51.639><c> going</c><01:34:51.719><c> to</c><01:34:52.080><c> be</c><01:34:52.239><c> more</c><01:34:52.480><c> around</c>\nI think you're going to be more around\nI think you're going to be more around like<01:34:52.840><c> the</c><01:34:53.000><c> 1</c><01:34:53.159><c> million</c>\nlike the 1 million\nlike the 1 million uh<01:34:54.440><c> order</c><01:34:54.719><c> of</c><01:34:54.880><c> magnitude</c><01:34:55.360><c> it's</c><01:34:55.480><c> still</c><01:34:55.719><c> much</c>\nuh order of magnitude it's still much\nuh order of magnitude it's still much less<01:34:56.159><c> than</c><01:34:56.320><c> pre-training</c><01:34:56.920><c> though</c><01:34:57.520><c> yeah</c>\nless than pre-training though yeah\nless than pre-training though yeah because<01:34:58.199><c> pre-training</c><01:34:58.639><c> is</c><01:34:58.800><c> 15</c><01:34:59.199><c> trillion</c>\nbecause pre-training is 15 trillion\nbecause pre-training is 15 trillion tokens<01:35:00.239><c> I</c><01:35:00.320><c> mean</c><01:35:00.520><c> this</c><01:35:00.639><c> is</c><01:35:01.000><c> like</c><01:35:01.480><c> that's</c><01:35:01.719><c> not</c>\ntokens I mean this is like that's not\ntokens I mean this is like that's not even<01:35:02.040><c> a</c><01:35:02.159><c> drop</c><01:35:02.679><c> and</c><01:35:02.920><c> yet</c><01:35:03.239><c> you</c><01:35:03.520><c> influence</c><01:35:04.000><c> the</c>\neven a drop and yet you influence the\neven a drop and yet you influence the weight<01:35:04.440><c> a</c><01:35:04.600><c> lot</c><01:35:05.000><c> so</c><01:35:05.360><c> because</c><01:35:05.520><c> you</c><01:35:05.679><c> do</c><01:35:05.800><c> it</c><01:35:05.960><c> I</c><01:35:06.000><c> mean</c>\nweight a lot so because you do it I mean\nweight a lot so because you do it I mean you<01:35:06.239><c> have</c><01:35:06.360><c> to</c><01:35:06.520><c> think</c><01:35:06.719><c> that</c><01:35:06.920><c> how</c><01:35:07.080><c> you</c><01:35:07.199><c> do</c><01:35:07.320><c> it</c><01:35:07.920><c> is</c>\nyou have to think that how you do it is\nyou have to think that how you do it is you<01:35:08.920><c> use</c><01:35:09.920><c> um</c><01:35:10.679><c> I</c><01:35:10.760><c> mean</c><01:35:11.080><c> as</c><01:35:11.199><c> I</c><01:35:11.320><c> said</c><01:35:11.560><c> the</c><01:35:12.000><c> learning</c>\nyou use um I mean as I said the learning\nyou use um I mean as I said the learning rate<01:35:12.480><c> that</c><01:35:12.560><c> you're</c><01:35:12.679><c> going</c><01:35:12.760><c> to</c><01:35:12.840><c> use</c><01:35:13.000><c> is</c><01:35:13.119><c> going</c>\nrate that you're going to use is going\nrate that you're going to use is going to<01:35:13.280><c> be</c><01:35:13.400><c> different</c><01:35:14.119><c> but</c><01:35:14.320><c> also</c><01:35:15.159><c> you</c><01:35:15.360><c> only</c><01:35:15.679><c> do</c>\nto be different but also you only do\nto be different but also you only do that<01:35:16.199><c> so</c><01:35:16.679><c> just</c><01:35:16.880><c> imagine</c><01:35:17.199><c> if</c><01:35:17.320><c> I</c><01:35:17.440><c> train</c><01:35:18.119><c> even</c><01:35:18.320><c> if</c>\nthat so just imagine if I train even if\nthat so just imagine if I train even if I<01:35:18.520><c> train</c><01:35:18.760><c> on</c><01:35:18.920><c> one</c><01:35:19.119><c> sentence</c><01:35:20.119><c> but</c><01:35:20.400><c> over</c><01:35:20.600><c> and</c>\nI train on one sentence but over and\nI train on one sentence but over and over<01:35:21.080><c> again</c><01:35:21.679><c> all</c><01:35:22.159><c> at</c><01:35:22.320><c> some</c><01:35:22.480><c> point</c><01:35:22.719><c> my</c><01:35:22.840><c> model</c>\nover again all at some point my model\nover again all at some point my model will<01:35:23.239><c> only</c><01:35:23.960><c> that</c><01:35:24.080><c> sentence</c><01:35:24.679><c> even</c><01:35:25.000><c> if</c><01:35:25.960><c> uh</c><01:35:26.199><c> it</c>\nwill only that sentence even if uh it\nwill only that sentence even if uh it was<01:35:26.520><c> just</c><01:35:26.679><c> one</c><01:35:26.880><c> sentence</c><01:35:27.360><c> instead</c><01:35:27.639><c> of</c><01:35:27.760><c> the</c><01:35:27.880><c> 15</c>\nwas just one sentence instead of the 15\nwas just one sentence instead of the 15 trillion<01:35:28.520><c> tokens</c><01:35:29.159><c> so</c><01:35:29.320><c> if</c><01:35:29.440><c> you</c><01:35:29.600><c> use</c><01:35:29.840><c> a</c><01:35:30.040><c> large</c>\ntrillion tokens so if you use a large\ntrillion tokens so if you use a large enough<01:35:30.600><c> learning</c><01:35:30.920><c> rate</c><01:35:31.400><c> and</c><01:35:31.520><c> for</c><01:35:32.040><c> enough</c><01:35:32.400><c> time</c>\nenough learning rate and for enough time\nenough learning rate and for enough time you<01:35:33.000><c> will</c><01:35:33.239><c> basically</c><01:35:33.800><c> overfit</c><01:35:34.320><c> that</c><01:35:34.480><c> sentence</c>\nyou will basically overfit that sentence\nyou will basically overfit that sentence so<01:35:35.400><c> the</c><01:35:35.600><c> the</c><01:35:35.760><c> the</c><01:35:35.920><c> key</c><01:35:36.119><c> thing</c><01:35:36.280><c> to</c><01:35:36.560><c> to</c><01:35:36.760><c> remember</c>\nso the the the key thing to to remember\nso the the the key thing to to remember is<01:35:37.360><c> that</c><01:35:38.159><c> um</c><01:35:38.760><c> the</c><01:35:38.920><c> data</c><01:35:39.159><c> is</c><01:35:39.280><c> not</c><01:35:39.440><c> I</c><01:35:39.760><c> it's</c><01:35:39.880><c> not</c><01:35:40.040><c> as</c>\nis that um the data is not I it's not as\nis that um the data is not I it's not as if<01:35:40.320><c> you</c><01:35:40.520><c> mix</c><01:35:41.280><c> some</c><01:35:41.520><c> posttraining</c><01:35:41.960><c> data</c><01:35:42.560><c> and</c>\nif you mix some posttraining data and\nif you mix some posttraining data and some<01:35:42.880><c> pre-training</c><01:35:43.400><c> data</c><01:35:43.800><c> you</c><01:35:43.920><c> do</c>\nsome pre-training data you do\nsome pre-training data you do pre-training<01:35:44.960><c> and</c><01:35:45.119><c> then</c><01:35:45.280><c> you</c><01:35:45.719><c> just</c><01:35:45.880><c> start</c>\npre-training and then you just start\npre-training and then you just start fine-tuning<01:35:47.040><c> only</c><01:35:47.239><c> on</c><01:35:47.360><c> the</c><01:35:47.480><c> post</c><01:35:47.719><c> trining</c><01:35:48.119><c> so</c>\nfine-tuning only on the post trining so\nfine-tuning only on the post trining so another<01:35:48.600><c> way</c><01:35:49.119><c> maybe</c><01:35:49.360><c> another</c><01:35:49.639><c> perspective</c><01:35:50.400><c> is</c>\nanother way maybe another perspective is\nanother way maybe another perspective is that<01:35:50.679><c> the</c><01:35:50.880><c> post</c><01:35:51.119><c> the</c><01:35:51.239><c> pre-training</c><01:35:52.080><c> is</c><01:35:52.199><c> just</c>\nthat the post the pre-training is just\nthat the post the pre-training is just the<01:35:52.560><c> initialization</c><01:35:53.159><c> of</c><01:35:53.280><c> your</c><01:35:53.400><c> model</c>\nthe initialization of your model\nthe initialization of your model and<01:35:54.280><c> once</c><01:35:54.440><c> you</c><01:35:54.600><c> view</c><01:35:54.800><c> it</c><01:35:55.000><c> that</c><01:35:55.199><c> way</c><01:35:55.520><c> that</c><01:35:55.719><c> this</c>\nand once you view it that way that this\nand once you view it that way that this is<01:35:56.000><c> just</c><01:35:56.199><c> initialization</c><01:35:56.800><c> of</c><01:35:57.080><c> Weights</c><01:35:58.080><c> then</c>\nis just initialization of Weights then\nis just initialization of Weights then there's<01:35:58.440><c> nothing</c><01:35:58.800><c> special</c><01:35:59.639><c> like</c><01:35:59.880><c> you</c><01:36:00.000><c> don't</c>\nthere's nothing special like you don't\nthere's nothing special like you don't need<01:36:00.360><c> to</c><01:36:00.600><c> remember</c><01:36:00.960><c> that</c><01:36:01.040><c> you</c><01:36:01.119><c> train</c><01:36:01.440><c> a</c><01:36:01.600><c> lot</c><01:36:01.719><c> of</c>\nneed to remember that you train a lot of\nneed to remember that you train a lot of data<01:36:02.159><c> before</c><01:36:02.639><c> the</c><01:36:02.760><c> only</c><01:36:02.920><c> thing</c><01:36:03.040><c> that</c><01:36:03.159><c> matters</c>\ndata before the only thing that matters\ndata before the only thing that matters is<01:36:03.600><c> that</c><01:36:03.679><c> you</c><01:36:03.760><c> had</c><01:36:03.880><c> an</c><01:36:04.000><c> initialization</c><01:36:05.000><c> and</c>\nis that you had an initialization and\nis that you had an initialization and now<01:36:05.280><c> I</c><01:36:05.440><c> actually</c><01:36:05.600><c> train</c><01:36:05.840><c> a</c><01:36:06.000><c> model</c><01:36:06.480><c> so</c><01:36:06.639><c> maybe</c>\nnow I actually train a model so maybe\nnow I actually train a model so maybe think<01:36:07.000><c> about</c><01:36:07.159><c> it</c><01:36:07.360><c> that</c><01:36:07.520><c> way</c><01:36:07.880><c> like</c><01:36:08.080><c> there's</c><01:36:08.360><c> a</c>\nthink about it that way like there's a\nthink about it that way like there's a there's<01:36:08.800><c> a</c><01:36:08.920><c> mark</c><01:36:09.119><c> of</c><01:36:09.280><c> property</c><01:36:09.920><c> in</c><01:36:10.119><c> some</c><01:36:10.360><c> way</c>\nthere's a mark of property in some way\nthere's a mark of property in some way just<01:36:10.760><c> like</c><01:36:10.880><c> you</c><01:36:11.000><c> had</c><01:36:11.159><c> your</c><01:36:11.320><c> weights</c><01:36:11.719><c> this</c><01:36:11.800><c> is</c>\njust like you had your weights this is\njust like you had your weights this is my<01:36:12.080><c> initialization</c><01:36:12.960><c> now</c><01:36:13.080><c> I'm</c><01:36:13.199><c> training</c><01:36:13.560><c> that</c>\nmy initialization now I'm training that\nmy initialization now I'm training that one<01:36:14.400><c> does</c><01:36:14.600><c> that</c><01:36:14.800><c> kind</c><01:36:14.880><c> of</c><01:36:15.040><c> answer</c><01:36:15.280><c> your</c>\none does that kind of answer your\none does that kind of answer your question<01:36:16.400><c> kind</c><01:36:16.560><c> of</c><01:36:17.400><c> but</c><01:36:18.400><c> you</c><01:36:18.520><c> said</c><01:36:18.760><c> something</c>\nquestion kind of but you said something\nquestion kind of but you said something just<01:36:19.280><c> now</c><01:36:19.639><c> about</c><01:36:20.480><c> it's</c><01:36:20.719><c> almost</c><01:36:21.000><c> the</c>\njust now about it's almost the\njust now about it's almost the equivalence<01:36:21.840><c> of</c><01:36:22.119><c> just</c><01:36:22.360><c> rerunning</c><01:36:22.920><c> the</c><01:36:23.280><c> find</c>\nequivalence of just rerunning the find\nequivalence of just rerunning the find tuning<01:36:23.880><c> data</c><01:36:24.199><c> many</c><01:36:24.520><c> times</c><01:36:25.239><c> is</c><01:36:25.320><c> it</c><01:36:25.560><c> actually</c><01:36:26.119><c> is</c>\ntuning data many times is it actually is\ntuning data many times is it actually is that<01:36:26.560><c> what</c><01:36:26.800><c> actually</c><01:36:27.159><c> happens</c><01:36:27.560><c> in</c><01:36:27.679><c> order</c><01:36:28.080><c> to</c>\nthat what actually happens in order to\nthat what actually happens in order to give<01:36:29.280><c> so</c><01:36:29.440><c> much</c><01:36:29.600><c> more</c><01:36:30.159><c> preference</c>\ngive so much more preference\ngive so much more preference um<01:36:33.840><c> you</c><01:36:34.239><c> might</c><01:36:34.920><c> I</c><01:36:35.080><c> actually</c><01:36:35.400><c> don't</c><01:36:35.679><c> know</c><01:36:36.000><c> right</c>\num you might I actually don't know right\num you might I actually don't know right now<01:36:36.360><c> how</c><01:36:36.520><c> they</c><01:36:36.639><c> do</c><01:36:36.800><c> it</c><01:36:36.880><c> in</c><01:36:37.040><c> Industry</c><01:36:37.719><c> when</c><01:36:37.960><c> we</c>\nnow how they do it in Industry when we\nnow how they do it in Industry when we did<01:36:38.320><c> alpaca</c><01:36:38.840><c> we</c><01:36:38.960><c> had</c><01:36:39.080><c> to</c><01:36:39.159><c> do</c><01:36:39.280><c> three</c><01:36:39.760><c> box</c><01:36:40.080><c> so</c><01:36:40.239><c> you</c>\ndid alpaca we had to do three box so you\ndid alpaca we had to do three box so you did<01:36:40.760><c> run</c><01:36:40.960><c> it</c><01:36:41.159><c> three</c><01:36:41.400><c> times</c><01:36:41.679><c> to</c><01:36:41.920><c> it</c>\ndid run it three times to it\ndid run it three times to it um<01:36:44.840><c> but</c><01:36:45.280><c> I</c><01:36:45.320><c> mean</c><01:36:45.520><c> even</c><01:36:45.679><c> the</c><01:36:45.840><c> number</c><01:36:46.000><c> of</c><01:36:46.159><c> times</c>\num but I mean even the number of times\num but I mean even the number of times that<01:36:46.560><c> you</c><01:36:46.679><c> run</c><01:36:46.920><c> it</c><01:36:47.080><c> through</c><01:36:47.360><c> it's</c><01:36:47.560><c> actually</c>\nthat you run it through it's actually\nthat you run it through it's actually not<01:36:48.040><c> important</c><01:36:48.560><c> the</c><01:36:48.679><c> only</c><01:36:48.920><c> thing</c><01:36:49.320><c> like</c><01:36:49.800><c> the</c>\nnot important the only thing like the\nnot important the only thing like the only<01:36:50.280><c> thing</c><01:36:50.440><c> is</c><01:36:50.600><c> the</c><01:36:51.040><c> is</c><01:36:51.159><c> kind</c><01:36:51.280><c> of</c><01:36:51.400><c> the</c>\nonly thing is the is kind of the\nonly thing is the is kind of the effective<01:36:51.840><c> learning</c><01:36:52.199><c> rate</c><01:36:52.639><c> that</c><01:36:52.880><c> what</c>\neffective learning rate that what\neffective learning rate that what matters\nmatters\nmatters um<01:36:54.880><c> so</c>\num so\num so yeah\nyeah\nyeah great<01:36:58.800><c> so</c><01:36:59.440><c> I</c><01:36:59.520><c> think</c><01:36:59.800><c> I</c><01:36:59.960><c> have</c><01:37:00.239><c> five</c><01:37:00.440><c> minutes</c>\n[Music]\n[Music]\n[Music] right<01:37:06.320><c> okay</c><01:37:07.080><c> I</c><01:37:08.080><c> might</c><01:37:08.920><c> try</c><01:37:09.560><c> to</c><01:37:09.920><c> give</c><01:37:10.080><c> a</c><01:37:10.320><c> high</c>\nright okay I might try to give a high\nright okay I might try to give a high level<01:37:10.920><c> Overview</c><01:37:11.800><c> at</c><01:37:11.920><c> least</c><01:37:12.119><c> from</c><01:37:12.400><c> one</c><01:37:12.600><c> of</c><01:37:12.760><c> the</c>\nlevel Overview at least from one of the\nlevel Overview at least from one of the systems<01:37:13.880><c> trick</c><01:37:14.880><c> systems</c><01:37:15.719><c> as</c><01:37:15.880><c> we</c><01:37:16.040><c> said</c><01:37:17.000><c> uh</c><01:37:17.320><c> for</c>\nsystems trick systems as we said uh for\nsystems trick systems as we said uh for everyone<01:37:18.119><c> Bott</c><01:37:18.440><c> neck</c><01:37:18.679><c> is</c><01:37:18.760><c> a</c><01:37:19.239><c> sorry</c><01:37:19.520><c> compute</c><01:37:19.880><c> is</c>\neveryone Bott neck is a sorry compute is\neveryone Bott neck is a sorry compute is the<01:37:20.159><c> huge</c><01:37:20.560><c> bottleneck</c><01:37:21.560><c> uh</c><01:37:21.679><c> one</c><01:37:21.880><c> question</c><01:37:22.080><c> you</c>\nthe huge bottleneck uh one question you\nthe huge bottleneck uh one question you might<01:37:22.400><c> ask</c><01:37:22.600><c> is</c><01:37:22.760><c> why</c><01:37:22.920><c> not</c><01:37:23.080><c> buy</c><01:37:23.239><c> more</c><01:37:23.760><c> gpus</c><01:37:24.760><c> uh</c>\nmight ask is why not buy more gpus uh\nmight ask is why not buy more gpus uh gpus<01:37:25.440><c> are</c><01:37:25.639><c> expensive</c><01:37:26.040><c> but</c><01:37:26.199><c> also</c><01:37:26.360><c> are</c><01:37:26.520><c> scarce</c>\ngpus are expensive but also are scarce\ngpus are expensive but also are scarce even<01:37:27.080><c> if</c><01:37:27.159><c> you</c><01:37:27.280><c> have</c><01:37:27.320><c> $10</c><01:37:27.560><c> million</c><01:37:28.159><c> right</c><01:37:28.280><c> now</c>\neven if you have $10 million right now\neven if you have $10 million right now you<01:37:28.679><c> cannot</c><01:37:29.040><c> buy</c><01:37:29.320><c> the</c><01:37:29.440><c> best</c><01:37:29.880><c> gpus</c><01:37:30.880><c> um</c>\nyou cannot buy the best gpus um\nyou cannot buy the best gpus um there's<01:37:33.280><c> oh</c><01:37:33.440><c> yeah</c><01:37:33.600><c> there's</c><01:37:33.800><c> also</c><01:37:34.040><c> some</c>\nthere's oh yeah there's also some\nthere's oh yeah there's also some physical<01:37:34.719><c> limitations</c><01:37:35.719><c> when</c><01:37:35.840><c> you</c><01:37:36.080><c> have</c><01:37:36.280><c> when</c>\nphysical limitations when you have when\nphysical limitations when you have when you<01:37:36.440><c> have</c><01:37:36.840><c> multiple</c><01:37:37.199><c> gpus</c><01:37:37.600><c> you</c><01:37:37.719><c> have</c><01:37:37.800><c> to</c>\nyou have multiple gpus you have to\nyou have multiple gpus you have to communicate<01:37:38.400><c> between</c><01:37:38.719><c> them</c><01:37:39.119><c> that</c><01:37:39.280><c> takes</c><01:37:39.639><c> time</c>\ncommunicate between them that takes time\ncommunicate between them that takes time um<01:37:40.760><c> so</c><01:37:41.080><c> just</c><01:37:41.239><c> buying</c><01:37:41.520><c> more</c><01:37:41.679><c> gpus</c><01:37:42.159><c> is</c><01:37:42.280><c> not</c><01:37:42.520><c> that</c>\num so just buying more gpus is not that\num so just buying more gpus is not that easy<01:37:43.679><c> um</c><01:37:43.880><c> so</c><01:37:44.080><c> it's</c><01:37:44.280><c> really</c><01:37:44.480><c> important</c><01:37:44.800><c> to</c>\neasy um so it's really important to\neasy um so it's really important to think<01:37:45.199><c> about</c><01:37:45.560><c> how</c><01:37:45.679><c> do</c><01:37:45.760><c> you</c><01:37:45.880><c> allocate</c>\nthink about how do you allocate\nthink about how do you allocate resources<01:37:46.760><c> and</c><01:37:46.880><c> how</c><01:37:47.000><c> do</c><01:37:47.080><c> you</c><01:37:47.199><c> optimize</c><01:37:47.560><c> your</c>\nresources and how do you optimize your\nresources and how do you optimize your pipeline<01:37:48.159><c> so</c><01:37:48.480><c> system</c><01:37:49.480><c> 101</c><01:37:50.280><c> on</c><01:37:50.880><c> gpus</c><01:37:51.440><c> I'm</c><01:37:51.560><c> sorry</c>\npipeline so system 101 on gpus I'm sorry\npipeline so system 101 on gpus I'm sorry I'm<01:37:51.880><c> going</c><01:37:52.239><c> slightly</c><01:37:52.679><c> faster</c><01:37:53.000><c> I</c><01:37:53.119><c> hope</c><01:37:53.280><c> for</c>\nI'm going slightly faster I hope for\nI'm going slightly faster I hope for that<01:37:53.679><c> some</c><01:37:53.840><c> of</c><01:37:53.960><c> you</c><01:37:54.199><c> at</c><01:37:54.320><c> least</c><01:37:54.520><c> can</c><01:37:54.719><c> follow</c><01:37:55.719><c> uh</c>\nthat some of you at least can follow uh\nthat some of you at least can follow uh gpus<01:37:56.280><c> are</c><01:37:56.440><c> basically</c><01:37:56.760><c> optimized</c><01:37:57.199><c> for</c>\ngpus are basically optimized for\ngpus are basically optimized for throughput<01:37:58.199><c> CPUs</c><01:37:58.840><c> are</c><01:37:59.280><c> optimized</c><01:38:00.280><c> uh</c><01:38:00.360><c> for</c>\nthroughput CPUs are optimized uh for\nthroughput CPUs are optimized uh for latency<01:38:01.599><c> so</c><01:38:02.000><c> gpus</c><01:38:02.480><c> the</c><01:38:02.599><c> way</c><01:38:02.719><c> you</c><01:38:02.840><c> have</c><01:38:02.920><c> to</c>\nlatency so gpus the way you have to\nlatency so gpus the way you have to think<01:38:03.239><c> about</c><01:38:03.440><c> it</c><01:38:03.599><c> is</c><01:38:03.719><c> that</c><01:38:03.920><c> there's</c><01:38:04.239><c> one</c><01:38:04.520><c> Comm</c>\nthink about it is that there's one Comm\nthink about it is that there's one Comm there's<01:38:05.239><c> one</c><01:38:05.480><c> command</c><01:38:05.840><c> that</c><01:38:05.960><c> is</c><01:38:06.119><c> run</c><01:38:06.679><c> on</c><01:38:07.000><c> many</c>\nthere's one command that is run on many\nthere's one command that is run on many many<01:38:07.480><c> Calles</c><01:38:07.800><c> at</c><01:38:07.920><c> the</c><01:38:08.040><c> same</c><01:38:08.239><c> time</c><01:38:08.719><c> on</c>\nmany Calles at the same time on\nmany Calles at the same time on different<01:38:09.239><c> type</c><01:38:09.480><c> of</c><01:38:09.679><c> data</c><01:38:10.639><c> um</c><01:38:11.520><c> so</c><01:38:12.000><c> this</c><01:38:12.080><c> is</c><01:38:12.320><c> how</c>\ndifferent type of data um so this is how\ndifferent type of data um so this is how you<01:38:12.520><c> see</c><01:38:12.679><c> a</c><01:38:12.800><c> GPU</c><01:38:13.239><c> you</c><01:38:13.320><c> see</c><01:38:13.520><c> there</c><01:38:13.639><c> are</c><01:38:13.920><c> many</c>\nyou see a GPU you see there are many\nyou see a GPU you see there are many different<01:38:14.400><c> CES</c><01:38:14.760><c> we</c><01:38:14.920><c> call</c><01:38:15.119><c> them</c><01:38:15.599><c> streaming</c>\ndifferent CES we call them streaming\ndifferent CES we call them streaming multiprocessors<01:38:17.480><c> which</c><01:38:17.599><c> is</c><01:38:17.760><c> very</c><01:38:17.920><c> different</c>\nmultiprocessors which is very different\nmultiprocessors which is very different than<01:38:18.280><c> the</c><01:38:18.440><c> usual</c><01:38:19.239><c> CPU</c><01:38:19.679><c> architecture</c><01:38:20.239><c> so</c><01:38:20.440><c> just</c>\nthan the usual CPU architecture so just\nthan the usual CPU architecture so just think<01:38:21.080><c> High</c><01:38:21.719><c> throughput</c><01:38:22.719><c> paralyzation</c><01:38:23.480><c> for</c>\nthink High throughput paralyzation for\nthink High throughput paralyzation for gpus<01:38:24.840><c> uh</c><01:38:24.960><c> gpus</c><01:38:25.360><c> are</c><01:38:25.520><c> optimized</c><01:38:25.920><c> for</c><01:38:26.119><c> fast</c>\ngpus uh gpus are optimized for fast\ngpus uh gpus are optimized for fast matrix<01:38:26.840><c> multiplication</c><01:38:27.840><c> so</c><01:38:28.400><c> every</c><01:38:28.679><c> time</c><01:38:28.840><c> you</c>\nmatrix multiplication so every time you\nmatrix multiplication so every time you will<01:38:29.119><c> do</c><01:38:29.560><c> uh</c><01:38:29.639><c> you</c><01:38:29.719><c> will</c><01:38:29.840><c> do</c><01:38:30.000><c> something</c><01:38:30.199><c> on</c><01:38:30.360><c> GPU</c>\nwill do uh you will do something on GPU\nwill do uh you will do something on GPU if<01:38:30.840><c> you</c><01:38:30.920><c> can</c><01:38:31.040><c> do</c><01:38:31.159><c> it</c><01:38:31.280><c> with</c><01:38:31.400><c> a</c><01:38:32.199><c> a</c><01:38:32.360><c> matrix</c>\nif you can do it with a a matrix\nif you can do it with a a matrix multiplication<01:38:33.440><c> it's</c><01:38:33.599><c> going</c><01:38:33.679><c> to</c><01:38:33.760><c> be</c><01:38:33.920><c> 10</c><01:38:34.159><c> times</c>\nmultiplication it's going to be 10 times\nmultiplication it's going to be 10 times faster<01:38:35.119><c> than</c><01:38:35.320><c> with</c><01:38:35.520><c> anything</c><01:38:35.800><c> else</c><01:38:36.719><c> uh</c><01:38:36.880><c> that</c>\nfaster than with anything else uh that\nfaster than with anything else uh that is<01:38:37.119><c> a</c><01:38:37.199><c> little</c><01:38:37.360><c> bit</c><01:38:37.480><c> annoying</c><01:38:37.920><c> because</c><01:38:38.040><c> it</c>\nis a little bit annoying because it\nis a little bit annoying because it means<01:38:38.360><c> that</c><01:38:38.520><c> we're</c><01:38:38.760><c> kind</c><01:38:38.920><c> of</c><01:38:39.800><c> uh</c><01:38:40.080><c> bottlenecked</c>\nmeans that we're kind of uh bottlenecked\nmeans that we're kind of uh bottlenecked to<01:38:40.920><c> doing</c><01:38:41.239><c> anything</c><01:38:41.599><c> with</c><01:38:42.119><c> Matrix</c>\nto doing anything with Matrix\nto doing anything with Matrix multiplications<01:38:44.119><c> um</c><01:38:44.360><c> another</c><01:38:44.639><c> thing</c><01:38:44.760><c> to</c><01:38:44.880><c> note</c>\nmultiplications um another thing to note\nmultiplications um another thing to note with<01:38:45.280><c> gpus</c><01:38:46.199><c> is</c><01:38:46.360><c> that</c><01:38:46.639><c> compute</c><01:38:47.480><c> has</c><01:38:47.599><c> been</c>\nwith gpus is that compute has been\nwith gpus is that compute has been improving<01:38:48.199><c> faster</c><01:38:48.560><c> than</c><01:38:48.760><c> memory</c><01:38:49.159><c> and</c>\nimproving faster than memory and\nimproving faster than memory and communication<01:38:50.480><c> so</c><01:38:50.880><c> right</c><01:38:51.080><c> now</c><01:38:51.840><c> gpus</c><01:38:52.840><c> usually</c>\ncommunication so right now gpus usually\ncommunication so right now gpus usually are<01:38:53.880><c> hard</c><01:38:54.159><c> to</c><01:38:54.679><c> keep</c><01:38:55.560><c> uh</c><01:38:56.000><c> like</c><01:38:56.119><c> the</c><01:38:56.280><c> data</c><01:38:56.520><c> that</c>\nare hard to keep uh like the data that\nare hard to keep uh like the data that you<01:38:56.719><c> send</c><01:38:57.000><c> that</c><01:38:57.400><c> send</c><01:38:57.639><c> to</c><01:38:57.800><c> gpus</c><01:38:58.719><c> is</c><01:38:58.920><c> actually</c>\nyou send that send to gpus is actually\nyou send that send to gpus is actually hard<01:38:59.400><c> to</c><01:38:59.560><c> keep</c><01:38:59.719><c> up</c><01:38:59.880><c> with</c><01:39:00.000><c> the</c><01:39:00.119><c> processess</c><01:39:00.760><c> so</c>\nhard to keep up with the processess so\nhard to keep up with the processess so most<01:39:01.159><c> of</c><01:39:01.280><c> your</c><01:39:01.440><c> gpus</c><01:39:01.840><c> are</c><01:39:02.000><c> actually</c><01:39:02.159><c> going</c><01:39:02.280><c> to</c>\nmost of your gpus are actually going to\nmost of your gpus are actually going to be<01:39:02.560><c> idle</c><01:39:03.040><c> if</c><01:39:03.159><c> you</c><01:39:03.280><c> just</c><01:39:03.440><c> run</c><01:39:03.719><c> normal</c><01:39:04.080><c> code</c><01:39:04.920><c> if</c>\nbe idle if you just run normal code if\nbe idle if you just run normal code if you<01:39:05.080><c> don't</c><01:39:05.280><c> optimize</c><01:39:05.679><c> your</c><01:39:05.840><c> code</c><01:39:06.320><c> so</c>\nyou don't optimize your code so\nyou don't optimize your code so communication<01:39:07.560><c> and</c><01:39:07.719><c> this</c><01:39:07.920><c> will</c><01:39:08.440><c> continue</c>\ncommunication and this will continue\ncommunication and this will continue over<01:39:10.119><c> time</c><01:39:11.119><c> another</c><01:39:11.400><c> thing</c><01:39:11.480><c> to</c><01:39:11.599><c> know</c><01:39:11.800><c> about</c>\nover time another thing to know about\nover time another thing to know about gpus<01:39:12.520><c> is</c><01:39:12.599><c> that</c><01:39:12.719><c> there's</c><01:39:12.840><c> a</c><01:39:13.000><c> memory</c><01:39:13.280><c> hierarchy</c>\ngpus is that there's a memory hierarchy\ngpus is that there's a memory hierarchy this<01:39:13.880><c> is</c><01:39:14.000><c> the</c><01:39:14.119><c> same</c><01:39:14.280><c> thing</c><01:39:14.440><c> actually</c><01:39:14.679><c> with</c>\nthis is the same thing actually with\nthis is the same thing actually with CPUs<01:39:15.520><c> but</c><01:39:15.679><c> basically</c><01:39:16.040><c> the</c><01:39:16.159><c> closer</c><01:39:16.800><c> you</c><01:39:16.920><c> are</c><01:39:17.080><c> to</c>\nCPUs but basically the closer you are to\nCPUs but basically the closer you are to your<01:39:17.400><c> cuse</c><01:39:17.760><c> the</c><01:39:17.880><c> less</c><01:39:18.040><c> memory</c><01:39:18.440><c> there</c><01:39:18.639><c> is</c><01:39:19.119><c> but</c>\nyour cuse the less memory there is but\nyour cuse the less memory there is but the<01:39:19.480><c> faster</c><01:39:19.840><c> things</c><01:39:20.080><c> run</c><01:39:20.679><c> if</c><01:39:20.800><c> you're</c><01:39:21.040><c> further</c>\nthe faster things run if you're further\nthe faster things run if you're further more<01:39:21.920><c> memory</c><01:39:22.360><c> slower</c>\nmore memory slower\nmore memory slower um<01:39:25.000><c> okay</c><01:39:25.119><c> I'm</c><01:39:25.239><c> going</c><01:39:25.320><c> to</c><01:39:25.400><c> skip</c><01:39:25.679><c> that</c><01:39:26.320><c> okay</c>\num okay I'm going to skip that okay\num okay I'm going to skip that okay actually<01:39:26.719><c> I'm</c><01:39:26.840><c> going</c><01:39:26.920><c> to</c><01:39:27.040><c> say</c><01:39:27.159><c> it</c><01:39:27.960><c> I</c><01:39:28.080><c> told</c><01:39:28.280><c> you</c>\nactually I'm going to say it I told you\nactually I'm going to say it I told you about<01:39:28.760><c> this</c><01:39:29.080><c> uh</c><01:39:29.239><c> the</c><01:39:29.440><c> fact</c><01:39:29.599><c> of</c><01:39:30.040><c> communication</c>\nabout this uh the fact of communication\nabout this uh the fact of communication uh<01:39:31.159><c> the</c><01:39:31.360><c> metric</c><01:39:31.639><c> that</c><01:39:31.760><c> people</c><01:39:31.960><c> usually</c><01:39:32.239><c> look</c>\nuh the metric that people usually look\nuh the metric that people usually look at<01:39:32.719><c> is</c><01:39:32.880><c> model</c><01:39:33.199><c> flop</c><01:39:33.560><c> utilization</c><01:39:34.440><c> so</c><01:39:34.599><c> what</c><01:39:34.719><c> is</c>\nat is model flop utilization so what is\nat is model flop utilization so what is the<01:39:35.040><c> theoretical</c><01:39:35.520><c> maximum</c><01:39:36.000><c> that</c><01:39:36.440><c> GPU</c><01:39:36.840><c> could</c>\nthe theoretical maximum that GPU could\nthe theoretical maximum that GPU could run<01:39:37.320><c> at</c><01:39:37.560><c> no</c><01:39:37.760><c> more</c><01:39:37.960><c> flops</c><01:39:38.280><c> that</c><01:39:38.360><c> you</c><01:39:38.440><c> could</c><01:39:38.560><c> use</c>\nrun at no more flops that you could use\nrun at no more flops that you could use per<01:39:38.960><c> second</c><01:39:39.880><c> divide</c><01:39:40.320><c> sorry</c><01:39:40.639><c> the</c><01:39:40.800><c> number</c><01:39:41.000><c> of</c><01:39:41.239><c> OB</c>\nper second divide sorry the number of OB\nper second divide sorry the number of OB observed<01:39:42.199><c> through</c><01:39:42.560><c> put</c><01:39:42.679><c> divided</c><01:39:43.000><c> by</c><01:39:43.119><c> this</c>\nobserved through put divided by this\nobserved through put divided by this theoretical<01:39:44.520><c> um</c><01:39:45.159><c> maximum</c><01:39:46.159><c> and</c><01:39:46.480><c> in</c><01:39:46.639><c> general</c><01:39:47.000><c> if</c>\ntheoretical um maximum and in general if\ntheoretical um maximum and in general if you<01:39:47.440><c> reach</c><01:39:47.760><c> 50%</c><01:39:48.320><c> you're</c><01:39:48.520><c> very</c><01:39:48.719><c> happy</c><01:39:49.440><c> like</c>\nyou reach 50% you're very happy like\nyou reach 50% you're very happy like Facebook<01:39:50.000><c> I</c><01:39:50.119><c> looked</c><01:39:50.320><c> at</c><01:39:50.440><c> Lama</c><01:39:50.840><c> was</c><01:39:50.960><c> at</c><01:39:51.119><c> 45</c><01:39:51.679><c> or</c>\nFacebook I looked at Lama was at 45 or\nFacebook I looked at Lama was at 45 or something<01:39:52.119><c> like</c><01:39:52.320><c> this</c><01:39:52.800><c> so</c><01:39:53.199><c> that</c><01:39:53.400><c> that</c><01:39:53.560><c> means</c>\nsomething like this so that that means\nsomething like this so that that means that<01:39:54.440><c> data</c><01:39:54.719><c> doesn't</c><01:39:55.000><c> come</c><01:39:55.280><c> fast</c><01:39:55.560><c> enough</c><01:39:56.000><c> even</c>\nthat data doesn't come fast enough even\nthat data doesn't come fast enough even for<01:39:56.480><c> these</c><01:39:56.679><c> big</c>\nfor these big\nfor these big companies<01:39:59.000><c> so</c><01:39:59.440><c> one</c><01:39:59.760><c> simple</c><01:40:00.080><c> trick</c><01:40:00.440><c> and</c><01:40:00.599><c> that</c>\ncompanies so one simple trick and that\ncompanies so one simple trick and that might<01:40:00.880><c> be</c><01:40:01.040><c> the</c><01:40:01.119><c> only</c><01:40:01.360><c> one</c><01:40:01.480><c> I'm</c><01:40:01.599><c> going</c><01:40:01.679><c> to</c><01:40:02.040><c> tell</c>\nmight be the only one I'm going to tell\nmight be the only one I'm going to tell you<01:40:02.320><c> about</c><01:40:02.800><c> is</c><01:40:02.960><c> low</c><01:40:03.480><c> Precision</c><01:40:04.480><c> one</c><01:40:04.760><c> simple</c>\nyou about is low Precision one simple\nyou about is low Precision one simple idea<01:40:05.840><c> is</c><01:40:06.040><c> that</c><01:40:06.320><c> well</c><01:40:06.599><c> if</c><01:40:06.760><c> I'm</c><01:40:06.880><c> going</c><01:40:07.000><c> to</c><01:40:07.159><c> put</c><01:40:07.400><c> my</c>\nidea is that well if I'm going to put my\nidea is that well if I'm going to put my floats<01:40:08.080><c> in</c><01:40:08.239><c> lower</c><01:40:08.599><c> Precision</c><01:40:09.480><c> then</c><01:40:09.639><c> there's</c>\nfloats in lower Precision then there's\nfloats in lower Precision then there's going<01:40:09.920><c> to</c><01:40:10.000><c> be</c><01:40:10.119><c> fewer</c><01:40:10.480><c> bits</c><01:40:10.800><c> that</c><01:40:10.880><c> I</c><01:40:11.000><c> have</c><01:40:11.119><c> to</c>\ngoing to be fewer bits that I have to\ngoing to be fewer bits that I have to send<01:40:11.480><c> to</c><01:40:11.639><c> my</c><01:40:11.760><c> gpus</c><01:40:12.480><c> if</c><01:40:12.639><c> there's</c><01:40:12.840><c> fewer</c><01:40:13.119><c> bits</c>\nsend to my gpus if there's fewer bits\nsend to my gpus if there's fewer bits it's<01:40:13.639><c> faster</c><01:40:13.920><c> communication</c><01:40:14.840><c> lower</c><01:40:15.119><c> memory</c>\nit's faster communication lower memory\nit's faster communication lower memory consumption<01:40:15.960><c> things</c><01:40:16.119><c> are</c><01:40:16.239><c> going</c><01:40:16.360><c> to</c><01:40:16.480><c> go</c>\nconsumption things are going to go\nconsumption things are going to go faster<01:40:17.800><c> uh</c><01:40:18.000><c> and</c><01:40:18.080><c> for</c><01:40:18.280><c> deep</c><01:40:18.520><c> learning</c><01:40:18.920><c> it</c><01:40:19.040><c> just</c>\nfaster uh and for deep learning it just\nfaster uh and for deep learning it just happens<01:40:19.560><c> that</c><01:40:20.000><c> de</c><01:40:20.480><c> decimal</c><01:40:21.280><c> is</c><01:40:21.440><c> not</c><01:40:21.679><c> that</c>\nhappens that de decimal is not that\nhappens that de decimal is not that important<01:40:22.880><c> uh</c><01:40:23.040><c> so</c><01:40:23.360><c> so</c><01:40:23.840><c> when</c><01:40:24.000><c> you</c><01:40:24.159><c> do</c><01:40:24.360><c> matrix</c>\nimportant uh so so when you do matrix\nimportant uh so so when you do matrix multiplication<01:40:25.599><c> when</c><01:40:25.719><c> you</c><01:40:25.840><c> do</c><01:40:26.040><c> like</c><01:40:26.159><c> for</c>\nmultiplication when you do like for\nmultiplication when you do like for example<01:40:26.560><c> SGD</c><01:40:27.000><c> there's</c><01:40:27.199><c> already</c><01:40:27.560><c> so</c><01:40:27.719><c> much</c>\nexample SGD there's already so much\nexample SGD there's already so much noise<01:40:28.560><c> that</c><01:40:28.719><c> if</c><01:40:28.800><c> you</c><01:40:28.960><c> update</c><01:40:29.320><c> something</c><01:40:29.639><c> by</c>\nnoise that if you update something by\nnoise that if you update something by 0.01<01:40:30.760><c> or</c>\n0.01 or\n0.01 or 0.015<01:40:32.880><c> who</c><01:40:33.040><c> cares</c><01:40:33.840><c> uh</c><01:40:33.920><c> so</c><01:40:34.119><c> basically</c><01:40:34.520><c> instead</c>\n0.015 who cares uh so basically instead\n0.015 who cares uh so basically instead of<01:40:34.960><c> using</c><01:40:35.960><c> uh</c><01:40:36.119><c> 32</c><01:40:36.639><c> bits</c><01:40:37.199><c> per</c><01:40:37.360><c> float</c><01:40:37.840><c> which</c><01:40:38.000><c> is</c>\nof using uh 32 bits per float which is\nof using uh 32 bits per float which is um<01:40:38.920><c> what</c><01:40:39.080><c> people</c><01:40:39.520><c> used</c><01:40:39.760><c> to</c><01:40:39.920><c> use</c><01:40:40.159><c> or</c><01:40:40.360><c> 64</c><01:40:40.920><c> for</c>\num what people used to use or 64 for\num what people used to use or 64 for example<01:40:41.400><c> which</c><01:40:41.480><c> is</c><01:40:41.639><c> what</c><01:40:41.840><c> you</c><01:40:42.000><c> would</c><01:40:42.199><c> use</c><01:40:42.639><c> in</c>\nexample which is what you would use in\nexample which is what you would use in other<01:40:43.119><c> domains</c><01:40:43.639><c> you</c><01:40:43.760><c> use</c><01:40:44.000><c> 16</c><01:40:44.440><c> bits</c><01:40:45.119><c> uh</c><01:40:45.239><c> for</c>\nother domains you use 16 bits uh for\nother domains you use 16 bits uh for matrix<01:40:45.760><c> multiplication</c><01:40:46.320><c> so</c><01:40:46.480><c> for</c><01:40:46.599><c> every</c><01:40:46.760><c> float</c>\nmatrix multiplication so for every float\nmatrix multiplication so for every float you<01:40:47.119><c> use</c><01:40:47.280><c> 16</c><01:40:48.000><c> bits</c><01:40:49.000><c> um</c><01:40:49.840><c> and</c><01:40:49.960><c> for</c><01:40:50.159><c> training</c><01:40:50.560><c> you</c>\nyou use 16 bits um and for training you\nyou use 16 bits um and for training you have<01:40:50.920><c> this</c><01:40:51.080><c> type</c><01:40:51.280><c> of</c><01:40:51.520><c> like</c><01:40:52.440><c> uh</c><01:40:52.599><c> what</c><01:40:52.719><c> we</c><01:40:52.840><c> call</c>\nhave this type of like uh what we call\nhave this type of like uh what we call aut<01:40:53.280><c> atic</c><01:40:53.520><c> mix</c><01:40:53.760><c> Precision</c><01:40:54.239><c> which</c><01:40:54.360><c> is</c><01:40:54.520><c> that</c><01:40:55.199><c> uh</c>\naut atic mix Precision which is that uh\naut atic mix Precision which is that uh some<01:40:55.599><c> of</c><01:40:55.719><c> the</c><01:40:55.840><c> things</c><01:40:56.080><c> are</c><01:40:56.199><c> in</c><01:40:56.360><c> 32</c><01:40:56.760><c> bits</c><01:40:57.199><c> others</c>\nsome of the things are in 32 bits others\nsome of the things are in 32 bits others are<01:40:57.599><c> in</c><01:40:57.719><c> 60</c><01:40:58.040><c> bit</c><01:40:58.679><c> in</c><01:40:58.840><c> 16</c><01:40:59.199><c> bits</c><01:41:00.040><c> um</c><01:41:00.320><c> generally</c>\nare in 60 bit in 16 bits um generally\nare in 60 bit in 16 bits um generally the<01:41:01.000><c> way</c><01:41:01.159><c> you</c><01:41:01.280><c> should</c><01:41:01.440><c> be</c><01:41:01.560><c> thinking</c><01:41:01.800><c> about</c><01:41:02.000><c> it</c>\nthe way you should be thinking about it\nthe way you should be thinking about it is<01:41:02.199><c> that</c><01:41:02.599><c> your</c><01:41:02.800><c> weights</c><01:41:03.639><c> are</c><01:41:03.880><c> stored</c><01:41:04.360><c> of</c><01:41:04.480><c> your</c>\nis that your weights are stored of your\nis that your weights are stored of your model<01:41:04.840><c> are</c><01:41:04.960><c> stored</c><01:41:05.199><c> in</c><01:41:05.320><c> 32</c><01:41:05.760><c> bits</c><01:41:06.679><c> um</c><01:41:07.159><c> but</c><01:41:07.400><c> just</c>\nmodel are stored in 32 bits um but just\nmodel are stored in 32 bits um but just before<01:41:07.840><c> the</c><01:41:08.000><c> computation</c><01:41:08.480><c> you</c><01:41:08.599><c> put</c>\nbefore the computation you put\nbefore the computation you put everything<01:41:09.040><c> in</c><01:41:09.280><c> 16</c><01:41:09.719><c> 16</c><01:41:10.119><c> bits</c><01:41:10.520><c> like</c><01:41:10.639><c> this</c><01:41:10.760><c> you</c>\neverything in 16 16 bits like this you\neverything in 16 16 bits like this you do<01:41:11.040><c> computation</c><01:41:11.639><c> super</c><01:41:11.960><c> fast</c><01:41:12.520><c> and</c><01:41:12.679><c> at</c><01:41:12.800><c> the</c><01:41:12.960><c> end</c>\ndo computation super fast and at the end\ndo computation super fast and at the end you<01:41:14.080><c> update</c><01:41:14.480><c> your</c><01:41:14.599><c> weights</c><01:41:15.119><c> in</c><01:41:15.320><c> 32</c><01:41:15.800><c> Bits</c><01:41:16.239><c> And</c>\nyou update your weights in 32 Bits And\nyou update your weights in 32 Bits And the<01:41:16.440><c> reason</c><01:41:16.679><c> why</c><01:41:16.800><c> you</c><01:41:16.920><c> do</c><01:41:17.040><c> all</c><01:41:17.199><c> the</c><01:41:17.360><c> updates</c><01:41:17.639><c> in</c>\nthe reason why you do all the updates in\nthe reason why you do all the updates in 32<01:41:18.159><c> bits</c><01:41:18.599><c> it's</c><01:41:18.760><c> just</c><01:41:19.000><c> think</c><01:41:19.320><c> that</c><01:41:19.400><c> if</c><01:41:19.520><c> your</c>\n32 bits it's just think that if your\n32 bits it's just think that if your learning<01:41:19.880><c> rate</c><01:41:20.080><c> for</c><01:41:20.199><c> example</c><01:41:20.480><c> is</c><01:41:20.639><c> very</c><01:41:20.840><c> small</c>\nlearning rate for example is very small\nlearning rate for example is very small you<01:41:21.599><c> still</c><01:41:21.800><c> want</c><01:41:21.920><c> to</c><01:41:22.080><c> be</c><01:41:22.239><c> able</c><01:41:22.480><c> to</c><01:41:22.719><c> like</c><01:41:23.000><c> make</c><01:41:23.400><c> a</c>\nyou still want to be able to like make a\nyou still want to be able to like make a difference<01:41:23.960><c> in</c><01:41:24.080><c> your</c><01:41:24.280><c> weights</c><01:41:25.040><c> uh</c><01:41:25.159><c> so</c><01:41:25.280><c> all</c><01:41:25.400><c> the</c>\ndifference in your weights uh so all the\ndifference in your weights uh so all the computation<01:41:26.440><c> is</c><01:41:26.719><c> done</c><01:41:27.199><c> in</c><01:41:27.360><c> 16</c><01:41:27.760><c> bits</c><01:41:28.400><c> but</c><01:41:28.800><c> the</c>\ncomputation is done in 16 bits but the\ncomputation is done in 16 bits but the weights<01:41:29.159><c> are</c><01:41:29.280><c> actually</c><01:41:29.440><c> stored</c><01:41:29.760><c> in</c><01:41:29.880><c> 32</c><01:41:30.320><c> bits</c>\nweights are actually stored in 32 bits\nweights are actually stored in 32 bits so<01:41:30.880><c> that's</c><01:41:31.119><c> like</c><01:41:31.280><c> the</c><01:41:31.400><c> standard</c><01:41:31.800><c> way</c><01:41:31.960><c> that</c>\nso that's like the standard way that\nso that's like the standard way that people<01:41:32.280><c> are</c><01:41:32.400><c> doing</c><01:41:33.000><c> it</c><01:41:34.199><c> um</c><01:41:35.199><c> okay</c><01:41:35.400><c> I'll</c>\npeople are doing it um okay I'll\npeople are doing it um okay I'll actually<01:41:35.880><c> talk</c><01:41:36.280><c> just</c><01:41:36.440><c> about</c><01:41:36.679><c> this</c><01:41:36.800><c> and</c><01:41:36.920><c> then</c>\nactually talk just about this and then\nactually talk just about this and then I'll<01:41:37.159><c> skip</c><01:41:37.360><c> all</c><01:41:37.480><c> the</c><01:41:37.599><c> rest</c><01:41:37.840><c> operator</c><01:41:38.280><c> Fusion</c>\nI'll skip all the rest operator Fusion\nI'll skip all the rest operator Fusion because<01:41:38.719><c> I</c><01:41:38.800><c> think</c><01:41:38.920><c> this</c><01:41:39.000><c> is</c><01:41:39.119><c> actually</c><01:41:39.400><c> pretty</c>\nbecause I think this is actually pretty\nbecause I think this is actually pretty cool<01:41:40.400><c> as</c><01:41:40.520><c> I</c><01:41:40.639><c> just</c><01:41:40.800><c> said</c><01:41:41.040><c> communication</c><01:41:41.599><c> is</c>\ncool as I just said communication is\ncool as I just said communication is very<01:41:42.040><c> slow</c><01:41:42.880><c> and</c><01:41:43.080><c> actually</c><01:41:43.520><c> every</c><01:41:43.800><c> time</c><01:41:44.199><c> you</c>\nvery slow and actually every time you\nvery slow and actually every time you use<01:41:44.560><c> a</c><01:41:44.760><c> pie</c><01:41:44.960><c> torch</c><01:41:45.239><c> line</c><01:41:45.920><c> it</c><01:41:46.119><c> basically</c><01:41:46.480><c> moves</c>\nuse a pie torch line it basically moves\nuse a pie torch line it basically moves variable<01:41:47.280><c> to</c><01:41:47.440><c> Global</c><01:41:47.760><c> memory</c><01:41:48.040><c> of</c><01:41:48.159><c> your</c><01:41:48.320><c> GPU</c><01:41:49.040><c> so</c>\nvariable to Global memory of your GPU so\nvariable to Global memory of your GPU so when<01:41:49.320><c> you</c><01:41:49.440><c> have</c><01:41:49.639><c> something</c><01:41:49.920><c> like</c><01:41:50.159><c> this</c><01:41:50.760><c> x</c><01:41:51.360><c> do</c>\nwhen you have something like this x do\nwhen you have something like this x do cosine<01:41:53.400><c> uh</c><01:41:53.520><c> equal</c><01:41:53.880><c> X1</c><01:41:54.360><c> and</c><01:41:54.480><c> then</c><01:41:54.599><c> you</c><01:41:55.080><c> do</c><01:41:55.239><c> X1</c><01:41:55.679><c> do</c>\ncosine uh equal X1 and then you do X1 do\ncosine uh equal X1 and then you do X1 do cosine<01:41:56.440><c> what</c><01:41:56.560><c> is</c><01:41:56.719><c> happening</c><01:41:57.159><c> behind</c><01:41:57.480><c> the</c>\ncosine what is happening behind the\ncosine what is happening behind the scenes<01:41:58.280><c> is</c><01:41:58.400><c> that</c><01:41:58.560><c> you</c><01:41:58.679><c> take</c><01:41:58.880><c> the</c><01:41:59.080><c> X</c><01:41:59.320><c> which</c><01:41:59.440><c> is</c>\nscenes is that you take the X which is\nscenes is that you take the X which is data<01:42:00.199><c> you</c><01:42:00.360><c> ship</c><01:42:00.639><c> it</c><01:42:00.800><c> to</c><01:42:01.000><c> your</c><01:42:01.719><c> um</c><01:42:01.880><c> to</c><01:42:02.040><c> your</c>\ndata you ship it to your um to your\ndata you ship it to your um to your actual<01:42:02.639><c> processes</c><01:42:03.080><c> of</c><01:42:03.239><c> your</c><01:42:03.360><c> gpus</c><01:42:03.960><c> you</c><01:42:04.119><c> apply</c>\nactual processes of your gpus you apply\nactual processes of your gpus you apply the<01:42:04.560><c> coign</c><01:42:05.119><c> you</c><01:42:05.239><c> ship</c><01:42:05.480><c> it</c><01:42:05.679><c> back</c><01:42:05.800><c> to</c><01:42:05.920><c> the</c><01:42:06.040><c> main</c>\nthe coign you ship it back to the main\nthe coign you ship it back to the main memory<01:42:06.639><c> of</c><01:42:06.719><c> your</c><01:42:06.880><c> GPU</c><01:42:07.719><c> and</c><01:42:07.880><c> then</c><01:42:08.080><c> you</c><01:42:08.280><c> see</c><01:42:08.560><c> the</c>\nmemory of your GPU and then you see the\nmemory of your GPU and then you see the next<01:42:08.920><c> sign</c><01:42:09.320><c> you</c><01:42:09.400><c> ship</c><01:42:09.639><c> it</c><01:42:09.840><c> back</c><01:42:09.960><c> to</c><01:42:10.119><c> the</c>\nnext sign you ship it back to the\nnext sign you ship it back to the computer<01:42:10.800><c> to</c><01:42:11.119><c> the</c><01:42:11.440><c> GPU</c><01:42:11.920><c> processor</c><01:42:12.480><c> you</c><01:42:12.639><c> apply</c>\ncomputer to the GPU processor you apply\ncomputer to the GPU processor you apply another<01:42:13.159><c> cosign</c><01:42:13.760><c> and</c><01:42:13.840><c> you</c><01:42:13.920><c> ship</c><01:42:14.159><c> it</c><01:42:14.280><c> back</c>\nanother cosign and you ship it back\nanother cosign and you ship it back again<01:42:15.440><c> um</c><01:42:15.960><c> so</c><01:42:16.239><c> another</c><01:42:16.520><c> way</c><01:42:16.639><c> to</c><01:42:16.760><c> see</c><01:42:17.000><c> that</c><01:42:17.199><c> is</c>\nagain um so another way to see that is\nagain um so another way to see that is that<01:42:17.440><c> you</c><01:42:17.639><c> go</c><01:42:17.800><c> from</c><01:42:17.960><c> your</c><01:42:18.159><c> Dam</c><01:42:18.639><c> which</c><01:42:18.719><c> is</c><01:42:18.800><c> your</c>\nthat you go from your Dam which is your\nthat you go from your Dam which is your Global<01:42:19.280><c> memory</c><01:42:19.880><c> in</c><01:42:20.000><c> your</c><01:42:20.159><c> GPU</c><01:42:20.960><c> and</c><01:42:21.080><c> you</c><01:42:21.239><c> ship</c>\nGlobal memory in your GPU and you ship\nGlobal memory in your GPU and you ship it<01:42:21.639><c> to</c><01:42:21.880><c> compute</c><01:42:22.400><c> you</c><01:42:22.480><c> ship</c><01:42:22.719><c> it</c><01:42:22.880><c> back</c><01:42:23.199><c> for</c><01:42:23.400><c> every</c>\nit to compute you ship it back for every\nit to compute you ship it back for every line<01:42:24.119><c> This</c><01:42:24.239><c> is</c><01:42:24.320><c> a</c><01:42:24.520><c> naive</c><01:42:24.880><c> way</c><01:42:25.000><c> of</c><01:42:25.159><c> doing</c><01:42:25.360><c> it</c>\nline This is a naive way of doing it\nline This is a naive way of doing it this<01:42:26.080><c> seems</c><01:42:26.440><c> very</c><01:42:26.880><c> wasteful</c><01:42:27.880><c> um</c><01:42:28.520><c> so</c><01:42:29.080><c> the</c><01:42:29.280><c> idea</c>\nthis seems very wasteful um so the idea\nthis seems very wasteful um so the idea simple<01:42:30.320><c> idea</c><01:42:30.679><c> of</c><01:42:30.880><c> operative</c><01:42:31.360><c> Fusion</c><01:42:31.880><c> is</c><01:42:32.040><c> just</c>\nsimple idea of operative Fusion is just\nsimple idea of operative Fusion is just communicate<01:42:33.320><c> do</c><01:42:33.520><c> all</c><01:42:33.679><c> the</c><01:42:33.800><c> computation</c><01:42:34.679><c> ship</c>\ncommunicate do all the computation ship\ncommunicate do all the computation ship it<01:42:35.119><c> back</c><01:42:35.360><c> once</c><01:42:36.199><c> and</c><01:42:36.400><c> this</c><01:42:36.520><c> is</c><01:42:36.719><c> exactly</c><01:42:37.199><c> what</c>\nit back once and this is exactly what\nit back once and this is exactly what fuse<01:42:37.960><c> kernels</c><01:42:38.480><c> are</c><01:42:39.360><c> um</c><01:42:39.560><c> so</c><01:42:39.760><c> if</c><01:42:39.840><c> you</c><01:42:40.080><c> ever</c><01:42:40.320><c> want</c>\nfuse kernels are um so if you ever want\nfuse kernels are um so if you ever want to<01:42:41.320><c> make</c><01:42:41.599><c> your</c><01:42:41.840><c> comp</c><01:42:42.520><c> your</c><01:42:43.520><c> computations</c><01:42:44.040><c> in</c>\nto make your comp your computations in\nto make your comp your computations in pytorch<01:42:44.760><c> much</c><01:42:45.000><c> faster</c><01:42:45.840><c> just</c><01:42:46.080><c> apply</c><01:42:46.400><c> torch.</c>\npytorch much faster just apply torch.\npytorch much faster just apply torch. compile<01:42:47.920><c> on</c><01:42:48.119><c> your</c><01:42:48.320><c> model</c><01:42:48.960><c> this</c><01:42:49.080><c> is</c><01:42:49.560><c> going</c><01:42:49.679><c> to</c>\ncompile on your model this is going to\ncompile on your model this is going to make<01:42:50.080><c> your</c><01:42:50.239><c> model</c><01:42:50.520><c> around</c><01:42:50.840><c> two</c><01:42:51.080><c> times</c><01:42:51.400><c> faster</c>\nmake your model around two times faster\nmake your model around two times faster and<01:42:52.280><c> what</c><01:42:52.400><c> it</c><01:42:52.560><c> does</c><01:42:52.920><c> is</c><01:42:53.280><c> simply</c><01:42:53.560><c> that</c><01:42:53.679><c> it</c>\nand what it does is simply that it\nand what it does is simply that it rewrites<01:42:54.800><c> your</c><01:42:55.199><c> code</c><01:42:56.199><c> uh</c><01:42:56.360><c> your</c><01:42:56.599><c> P</c><01:42:56.920><c> like</c><01:42:57.040><c> your</c>\nrewrites your code uh your P like your\nrewrites your code uh your P like your py<01:42:57.440><c> torch</c><01:42:57.719><c> code</c><01:42:58.320><c> basically</c><01:42:59.159><c> in</c><01:42:59.360><c> C++</c><01:43:00.119><c> in</c><01:43:00.440><c> Cuda</c>\npy torch code basically in C++ in Cuda\npy torch code basically in C++ in Cuda uh<01:43:01.679><c> to</c><01:43:02.639><c> to</c><01:43:02.920><c> do</c><01:43:03.080><c> the</c><01:43:03.239><c> communication</c><01:43:03.679><c> only</c><01:43:03.920><c> once</c>\nuh to to do the communication only once\nuh to to do the communication only once then<01:43:04.400><c> do</c><01:43:04.560><c> all</c><01:43:04.719><c> the</c><01:43:04.920><c> operations</c><01:43:05.560><c> then</c><01:43:06.040><c> uh</c><01:43:06.159><c> ship</c>\nthen do all the operations then uh ship\nthen do all the operations then uh ship it<01:43:07.000><c> back</c><01:43:08.000><c> okay</c><01:43:08.239><c> I'm</c><01:43:08.400><c> not</c><01:43:08.520><c> going</c><01:43:08.639><c> to</c><01:43:08.760><c> have</c><01:43:08.920><c> time</c>\nit back okay I'm not going to have time\nit back okay I'm not going to have time to<01:43:09.239><c> talk</c><01:43:09.400><c> about</c><01:43:09.599><c> tiling</c><01:43:10.400><c> tiling</c><01:43:10.719><c> is</c><01:43:10.920><c> important</c>\nto talk about tiling tiling is important\nto talk about tiling tiling is important paration<01:43:12.639><c> paration</c><01:43:13.199><c> is</c><01:43:13.639><c> important</c><01:43:14.840><c> um</c><01:43:15.840><c> and</c>\nparation paration is important um and\nparation paration is important um and mixture<01:43:16.440><c> of</c><01:43:16.599><c> experts</c><01:43:17.159><c> mixture</c><01:43:17.440><c> of</c><01:43:17.560><c> experts</c><01:43:18.000><c> is</c>\nmixture of experts mixture of experts is\nmixture of experts mixture of experts is important<01:43:18.920><c> Outlook</c><01:43:19.880><c> there</c><01:43:20.000><c> are</c><01:43:20.119><c> many</c><01:43:20.320><c> things</c>\nimportant Outlook there are many things\nimportant Outlook there are many things we<01:43:20.639><c> haven't</c><01:43:21.000><c> T</c><01:43:22.000><c> talked</c><01:43:22.320><c> about</c><01:43:23.239><c> we</c><01:43:23.360><c> haven't</c>\nwe haven't T talked about we haven't\nwe haven't T talked about we haven't talked<01:43:23.800><c> about</c><01:43:24.280><c> architectures</c><01:43:24.920><c> we</c><01:43:25.080><c> definitely</c>\ntalked about architectures we definitely\ntalked about architectures we definitely haven't<01:43:25.599><c> talked</c><01:43:25.800><c> about</c><01:43:26.280><c> inference</c><01:43:27.280><c> um</c><01:43:27.639><c> there</c>\nhaven't talked about inference um there\nhaven't talked about inference um there are<01:43:27.880><c> many</c><01:43:28.119><c> other</c><01:43:28.280><c> things</c><01:43:28.440><c> that</c><01:43:28.560><c> are</c><01:43:28.719><c> important</c>\nare many other things that are important\nare many other things that are important with<01:43:29.199><c> LMS</c><01:43:30.000><c> what</c><01:43:30.119><c> is</c><01:43:30.239><c> the</c><01:43:30.400><c> UI</c><01:43:30.760><c> that</c><01:43:30.880><c> you</c><01:43:31.080><c> use</c><01:43:31.320><c> I</c>\nwith LMS what is the UI that you use I\nwith LMS what is the UI that you use I mean<01:43:31.880><c> arguably</c><01:43:32.360><c> chat</c><01:43:32.599><c> jpt</c><01:43:33.000><c> the</c><01:43:33.159><c> big</c><01:43:33.320><c> novelty</c>\nmean arguably chat jpt the big novelty\nmean arguably chat jpt the big novelty was<01:43:33.960><c> just</c><01:43:34.320><c> have</c><01:43:34.480><c> a</c><01:43:34.599><c> simple</c><01:43:34.880><c> UI</c><01:43:35.199><c> to</c><01:43:35.320><c> use</c><01:43:35.480><c> it</c>\nwas just have a simple UI to use it\nwas just have a simple UI to use it multimodality<01:43:36.880><c> what</c><01:43:37.000><c> are</c><01:43:37.119><c> all</c><01:43:37.239><c> the</c><01:43:37.360><c> misuses</c>\nmultimodality what are all the misuses\nmultimodality what are all the misuses you<01:43:37.920><c> could</c><01:43:38.159><c> have</c><01:43:38.840><c> uh</c><01:43:38.960><c> the</c><01:43:39.119><c> fact</c><01:43:39.280><c> that</c><01:43:39.400><c> there</c>\nyou could have uh the fact that there\nyou could have uh the fact that there might<01:43:39.719><c> not</c><01:43:39.840><c> be</c><01:43:40.000><c> enough</c><01:43:40.239><c> data</c><01:43:40.440><c> on</c><01:43:40.560><c> the</c><01:43:40.719><c> internet</c>\nmight not be enough data on the internet\nmight not be enough data on the internet to<01:43:41.159><c> train</c><01:43:41.400><c> all</c><01:43:41.560><c> these</c><01:43:41.719><c> models</c><01:43:42.440><c> legality</c><01:43:42.920><c> of</c>\nto train all these models legality of\nto train all these models legality of data<01:43:43.280><c> collection</c><01:43:43.920><c> so</c><01:43:44.119><c> many</c><01:43:44.400><c> other</c><01:43:44.639><c> things</c><01:43:45.320><c> if</c>\ndata collection so many other things if\ndata collection so many other things if you<01:43:45.560><c> are</c><01:43:45.760><c> interested</c><01:43:46.119><c> in</c><01:43:46.280><c> all</c><01:43:46.440><c> these</c><01:43:46.679><c> topics</c>\nyou are interested in all these topics\nyou are interested in all these topics uh<01:43:47.840><c> I</c><01:43:47.920><c> would</c><01:43:48.119><c> suggest</c><01:43:48.520><c> three</c><01:43:48.760><c> classes</c><01:43:49.840><c> cs224n</c>\nuh I would suggest three classes cs224n\nuh I would suggest three classes cs224n is<01:43:51.000><c> probably</c><01:43:51.280><c> the</c><01:43:51.360><c> one</c><01:43:51.480><c> that</c><01:43:51.599><c> touches</c><01:43:51.960><c> the</c>\nis probably the one that touches the\nis probably the one that touches the least<01:43:52.719><c> on</c><01:43:53.400><c> uh</c><01:43:53.760><c> LMS</c><01:43:54.760><c> uh</c><01:43:54.880><c> but</c><01:43:55.000><c> it</c><01:43:55.119><c> gives</c><01:43:55.320><c> some</c>\nleast on uh LMS uh but it gives some\nleast on uh LMS uh but it gives some background<01:43:55.840><c> and</c><01:43:56.040><c> historical</c><01:43:56.639><c> context</c><01:43:57.639><c> um</c><01:43:58.040><c> of</c>\nbackground and historical context um of\nbackground and historical context um of all<01:43:58.520><c> the</c><01:43:58.679><c> LMS</c><01:43:59.119><c> and</c><01:43:59.280><c> gives</c><01:43:59.719><c> kind</c><01:43:59.840><c> of</c><01:43:59.960><c> some</c>\nall the LMS and gives kind of some\nall the LMS and gives kind of some adjacent<01:44:00.800><c> material</c><01:44:01.599><c> CS</c><01:44:02.000><c> 324</c><01:44:02.840><c> I</c><01:44:02.960><c> think</c><01:44:03.119><c> it's</c>\nadjacent material CS 324 I think it's\nadjacent material CS 324 I think it's called<01:44:04.040><c> Uh</c><01:44:05.040><c> I</c><01:44:05.159><c> think</c><01:44:05.280><c> it's</c><01:44:05.360><c> just</c><01:44:05.520><c> called</c><01:44:05.840><c> large</c>\ncalled Uh I think it's just called large\ncalled Uh I think it's just called large language<01:44:06.360><c> models</c><01:44:07.199><c> uh</c><01:44:07.360><c> more</c><01:44:07.599><c> in-depth</c><01:44:08.000><c> reading</c>\nlanguage models uh more in-depth reading\nlanguage models uh more in-depth reading and<01:44:08.440><c> lectures</c><01:44:08.920><c> on</c><01:44:09.239><c> everything</c><01:44:09.520><c> I</c><01:44:09.679><c> talked</c>\nand lectures on everything I talked\nand lectures on everything I talked about<01:44:10.280><c> CS</c><01:44:10.880><c> 336</c><01:44:11.880><c> which</c><01:44:12.000><c> is</c><01:44:12.280><c> large</c><01:44:12.560><c> language</c>\nabout CS 336 which is large language\nabout CS 336 which is large language model<01:44:13.199><c> from</c><01:44:13.400><c> scratch</c><01:44:13.960><c> you</c><01:44:14.280><c> actually</c><01:44:14.560><c> build</c>\nmodel from scratch you actually build\nmodel from scratch you actually build your<01:44:15.000><c> own</c><01:44:15.679><c> llm</c><01:44:16.679><c> uh</c><01:44:16.960><c> it's</c><01:44:17.320><c> an</c><01:44:17.480><c> amazing</c><01:44:17.960><c> class</c>\nyour own llm uh it's an amazing class\nyour own llm uh it's an amazing class also<01:44:18.719><c> given</c><01:44:19.040><c> by</c><01:44:19.239><c> my</c><01:44:19.360><c> two</c><01:44:19.679><c> supervisors</c><01:44:20.639><c> very</c>\nalso given by my two supervisors very\nalso given by my two supervisors very heavy<01:44:21.159><c> workload</c><01:44:21.639><c> so</c><01:44:21.800><c> be</c><01:44:21.920><c> careful</c><01:44:22.920><c> and</c><01:44:23.239><c> um</c>\nheavy workload so be careful and um\nheavy workload so be careful and um great",
+  "fetched_at": "2026-06-21T19:50:55Z",
+  "source": "yt-dlp-vtt"
+}
\ No newline at end of file
diff --git a/conductor/tracks/video_analysis_cs229_building_llms_20260621/artifacts/video.log b/conductor/tracks/video_analysis_cs229_building_llms_20260621/artifacts/video.log
new file mode 100644
index 00000000..1e2d48e5
--- /dev/null
+++ b/conductor/tracks/video_analysis_cs229_building_llms_20260621/artifacts/video.log
@@ -0,0 +1,10 @@
+# yt-dlp log
+# url: https://youtu.be/9vM4p9NN0Ts
+# output: C:\projects\manual_slop\conductor\tracks\video_analysis_cs229_building_llms_20260621\artifacts\video.mp4
+# returncode: 0
+
+stdout:
+
+
+stderr:
+
diff --git a/scripts/tier2/artifacts/video_analysis_campaign_20260621/phase1_acquire_cs229.py b/scripts/tier2/artifacts/video_analysis_campaign_20260621/phase1_acquire_cs229.py
new file mode 100644
index 00000000..30ee5f66
--- /dev/null
+++ b/scripts/tier2/artifacts/video_analysis_campaign_20260621/phase1_acquire_cs229.py
@@ -0,0 +1,117 @@
+"""Phase 1 Acquire driver for video_analysis_cs229_building_llms_20260621.
+
+Strategy: youtube-transcript-api fails for this video (R5: XML parse error on empty response,
+likely a YouTube API restriction). Fall back to yt-dlp's own subtitle extraction.
+"""
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[4]
+sys.path.insert(0, str(ROOT))
+
+from scripts.video_analysis.download_video import download_video
+from scripts.video_analysis.extract_transcript import _fetch_raw_transcript
+
+URL = "https://youtu.be/9vM4p9NN0Ts"
+ARTIFACTS = ROOT / "conductor" / "tracks" / "video_analysis_cs229_building_llms_20260621" / "artifacts"
+ARTIFACTS.mkdir(parents=True, exist_ok=True)
+
+
+def _parse_vtt_segments(vtt_path: Path) -> list[dict]:
+ text = vtt_path.read_text(encoding="utf-8")
+ segments: list[dict] = []
+ pattern = re.compile(r"(\d{2}):(\d{2}):(\d{2})\.(\d{3})\s+-->", re.MULTILINE)
+ blocks = re.split(r"\n\n+", text)
+ for block in blocks:
+  match = pattern.search(block)
+  if not match:
+   continue
+  h, m, s, ms = match.groups()
+  start = int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000.0
+  lines = [ln.strip() for ln in block.split("\n") if ln.strip() and not pattern.match(ln) and "-->" not in ln]
+  text_content = " ".join(lines)
+  if text_content:
+   segments.append({"start": start, "duration": 0.0, "text": text_content})
+ return segments
+
+
+def main() -> int:
+ print(f"Phase 1 Acquire for {URL}")
+ print(f"Artifacts: {ARTIFACTS}")
+
+ print("Step 1: extract_transcript (try youtube-transcript-api)")
+ transcript_path = ARTIFACTS / "transcript.json"
+ video_id = "9vM4p9NN0Ts"
+ last_exc = None
+ for attempt in range(3):
+  try:
+   segments = _fetch_raw_transcript(video_id)
+   data = {
+    "video_id": video_id,
+    "segments": segments,
+    "plain": "\n".join(s["text"] for s in segments),
+    "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+    "source": "youtube-transcript-api",
+   }
+   transcript_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+   print(f"  OK: wrote {transcript_path} ({len(segments)} segments)")
+   break
+  except Exception as e:
+   last_exc = e
+   print(f"  attempt {attempt+1} failed: {type(e).__name__}: {str(e)[:200]}")
+   if attempt < 2:
+    time.sleep(2 ** attempt)
+ else:
+  print(f"  youtube-transcript-api failed after 3 attempts. Falling back to yt-dlp subtitles.")
+  print("Step 1b: yt-dlp subtitle fallback")
+  vtt_path = ARTIFACTS / f"{video_id}.en.vtt"
+  completed = subprocess.run(
+   ["yt-dlp", "--write-auto-subs", "--sub-langs", "en", "--sub-format", "vtt",
+    "--skip-download", "--output", str(ARTIFACTS / video_id), URL],
+   capture_output=True, text=True,
+  )
+  candidates = list(ARTIFACTS.glob(f"{video_id}*.vtt"))
+  if not candidates:
+   print(f"  yt-dlp subtitle fetch also failed: {completed.stderr[:300]}")
+   print(f"  No transcript available. Continuing with download only.")
+   transcript_path.write_text(json.dumps({
+    "video_id": video_id,
+    "segments": [],
+    "plain": "",
+    "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+    "source": "unavailable",
+    "error": str(last_exc)[:500] if last_exc else None,
+    "note": "youtube-transcript-api failed with XML parse error (R5). yt-dlp subtitles also unavailable. Frame OCR will be the primary signal for this video.",
+   }, indent=2, ensure_ascii=False), encoding="utf-8")
+  else:
+   vtt_path = candidates[0]
+   segments = _parse_vtt_segments(vtt_path)
+   data = {
+    "video_id": video_id,
+    "segments": segments,
+    "plain": "\n".join(s["text"] for s in segments),
+    "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+    "source": "yt-dlp-vtt",
+   }
+   transcript_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+   print(f"  OK: parsed {len(segments)} segments from {vtt_path.name}")
+
+ print("Step 2: download_video")
+ video_path = ARTIFACTS / "video.mp4"
+ result = download_video(URL, video_path)
+ if result.is_err():
+  print(f"  ERR: {result.err.class_name}: {result.err.detail[:200]}")
+  return 1
+ print(f"  OK: wrote {video_path} ({video_path.stat().st_size} bytes)")
+ print(f"  log: {result.value['log']}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/video_analysis/extract_transcript.py b/scripts/video_analysis/extract_transcript.py
index c3c45dfb..81c019ae 100644
--- a/scripts/video_analysis/extract_transcript.py
+++ b/scripts/video_analysis/extract_transcript.py
@@ -70,10 +70,9 @@ def format_transcript_json(video_id: str, segments: list[dict[str, Any]]) -> dic
 
 
 def _fetch_raw_transcript(video_id: str) -> list[dict[str, Any]]:
- api = YouTubeTranscriptApi()
- fetched = api.fetch(video_id)
+ fetched = YouTubeTranscriptApi.get_transcript(video_id)
  return [
-  {"start": float(s.start), "duration": float(s.duration), "text": str(s.text)}
+  {"start": float(s["start"]), "duration": float(s["duration"]), "text": str(s["text"])}
   for s in fetched
  ]