{ "cells": [ { "cell_type": "code", "execution_count": 185, "id": "599306b1-777c-447a-915c-8b7753065da8", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "from sklearn.linear_model import LinearRegression\n", "from sklearn.metrics import mean_squared_error, r2_score\n", "from sklearn.ensemble import RandomForestRegressor\n", "from sklearn.model_selection import train_test_split" ] }, { "cell_type": "code", "execution_count": 91, "id": "16c87bdd-6384-4e6b-b9cc-2be6dcde0c63", "metadata": {}, "outputs": [], "source": [ "df = pd.read_csv(\"youtube.csv\")" ] }, { "cell_type": "code", "execution_count": 92, "id": "6a06c67b-44f7-4a3e-90f5-6ac5bfaed688", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
indexvideo_idtrending_datetitlechannel_titlecategory_idpublish_datetime_framepublished_day_of_weekpublish_countrytagsviewslikesdislikescomment_countcomments_disabledratings_disabledvideo_error_or_removed
002kyS6SvSYSE17.14.11WE WANT TO TALK ABOUT OUR MARRIAGECaseyNeistat2213/11/201717:00 to 17:59MondayUSSHANtell martin74837457527296615954FalseFalseFalse
111ZAPwfrtAFY17.14.11The Trump Presidency: Last Week Tonight with J...LastWeekTonight2413/11/20177:00 to 7:59MondayUSlast week tonight trump presidency|\"last week ...241878397185614612703FalseFalseFalse
225qpjK5DgCt417.14.11Racist Superman | Rudy Mancuso, King Bach & Le...Rudy Mancuso2312/11/201719:00 to 19:59SundayUSracist superman|\"rudy\"\"|\"\"mancuso\"\"|\"\"king\"\"|\"...319143414603353398181FalseFalseFalse
33puqaWrEC7tY17.14.11Nickelback Lyrics: Real or Fake?Good Mythical Morning2413/11/201711:00 to 11:59MondayUSrhett and link|\"gmm\"\"|\"\"good mythical morning\"...343168101726662146FalseFalseFalse
44d380meD0W0M17.14.11I Dare You: GOING BALD!?nigahiga2412/11/201718:00 to 18:59SundayUSryan|\"higa\"\"|\"\"higatv\"\"|\"\"nigahiga\"\"|\"\"i dare ...2095731132235198917518FalseFalseFalse
\n", "
" ], "text/plain": [ " index video_id trending_date \\\n", "0 0 2kyS6SvSYSE 17.14.11 \n", "1 1 1ZAPwfrtAFY 17.14.11 \n", "2 2 5qpjK5DgCt4 17.14.11 \n", "3 3 puqaWrEC7tY 17.14.11 \n", "4 4 d380meD0W0M 17.14.11 \n", "\n", " title channel_title \\\n", "0 WE WANT TO TALK ABOUT OUR MARRIAGE CaseyNeistat \n", "1 The Trump Presidency: Last Week Tonight with J... LastWeekTonight \n", "2 Racist Superman | Rudy Mancuso, King Bach & Le... Rudy Mancuso \n", "3 Nickelback Lyrics: Real or Fake? Good Mythical Morning \n", "4 I Dare You: GOING BALD!? nigahiga \n", "\n", " category_id publish_date time_frame published_day_of_week \\\n", "0 22 13/11/2017 17:00 to 17:59 Monday \n", "1 24 13/11/2017 7:00 to 7:59 Monday \n", "2 23 12/11/2017 19:00 to 19:59 Sunday \n", "3 24 13/11/2017 11:00 to 11:59 Monday \n", "4 24 12/11/2017 18:00 to 18:59 Sunday \n", "\n", " publish_country tags views \\\n", "0 US SHANtell martin 748374 \n", "1 US last week tonight trump presidency|\"last week ... 2418783 \n", "2 US racist superman|\"rudy\"\"|\"\"mancuso\"\"|\"\"king\"\"|\"... 3191434 \n", "3 US rhett and link|\"gmm\"\"|\"\"good mythical morning\"... 343168 \n", "4 US ryan|\"higa\"\"|\"\"higatv\"\"|\"\"nigahiga\"\"|\"\"i dare ... 2095731 \n", "\n", " likes dislikes comment_count comments_disabled ratings_disabled \\\n", "0 57527 2966 15954 False False \n", "1 97185 6146 12703 False False \n", "2 146033 5339 8181 False False \n", "3 10172 666 2146 False False \n", "4 132235 1989 17518 False False \n", "\n", " video_error_or_removed \n", "0 False \n", "1 False \n", "2 False \n", "3 False \n", "4 False " ] }, "execution_count": 92, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": 95, "id": "fd1e8adc-2ac0-4a3d-a57f-2c63ffa98245", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
indexcategory_idviewslikesdislikescomment_count
count161470.00000161470.0000001.614700e+051.614700e+051.614700e+051.614700e+05
mean80734.5000019.4611512.419854e+066.566194e+043.490153e+037.035494e+03
std46612.518327.4320011.043749e+072.260617e+053.114779e+043.404121e+04
min0.000001.0000002.230000e+020.000000e+000.000000e+000.000000e+00
25%40367.2500015.0000001.015382e+051.975000e+038.500000e+012.790000e+02
50%80734.5000023.0000003.847395e+059.840000e+033.480000e+021.144000e+03
75%121101.7500024.0000001.339528e+064.006275e+041.350000e+034.144750e+03
max161469.0000044.0000004.245389e+085.613827e+061.944971e+061.626501e+06
\n", "
" ], "text/plain": [ " index category_id views likes dislikes \\\n", "count 161470.00000 161470.000000 1.614700e+05 1.614700e+05 1.614700e+05 \n", "mean 80734.50000 19.461151 2.419854e+06 6.566194e+04 3.490153e+03 \n", "std 46612.51832 7.432001 1.043749e+07 2.260617e+05 3.114779e+04 \n", "min 0.00000 1.000000 2.230000e+02 0.000000e+00 0.000000e+00 \n", "25% 40367.25000 15.000000 1.015382e+05 1.975000e+03 8.500000e+01 \n", "50% 80734.50000 23.000000 3.847395e+05 9.840000e+03 3.480000e+02 \n", "75% 121101.75000 24.000000 1.339528e+06 4.006275e+04 1.350000e+03 \n", "max 161469.00000 44.000000 4.245389e+08 5.613827e+06 1.944971e+06 \n", "\n", " comment_count \n", "count 1.614700e+05 \n", "mean 7.035494e+03 \n", "std 3.404121e+04 \n", "min 0.000000e+00 \n", "25% 2.790000e+02 \n", "50% 1.144000e+03 \n", "75% 4.144750e+03 \n", "max 1.626501e+06 " ] }, "execution_count": 95, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.describe()" ] }, { "cell_type": "code", "execution_count": 97, "id": "75bc7548-b375-4c5d-85dd-54735230e98d", "metadata": {}, "outputs": [], "source": [ "df['publish_date'] = pd.to_datetime(df['publish_date'], dayfirst=True, errors='coerce')\n", "df['trending_date'] = pd.to_datetime(\n", " df['trending_date'],\n", " format='%y.%d.%m'\n", ")" ] }, { "cell_type": "code", "execution_count": 99, "id": "a2d7d781-3422-4052-9338-c8e4153ccaba", "metadata": {}, "outputs": [], "source": [ "df['tags_count'] = df['tags'].str.count('\\|') + 1\n", "df['tags_length'] = df['tags'].str.len()" ] }, { "cell_type": "code", "execution_count": 101, "id": "fd28be6c-d3db-46ef-944d-2ee791c1399a", "metadata": {}, "outputs": [], "source": [ "df['publish_hour'] = df['time_frame'].str.split(':').str[0].astype(int)" ] }, { "cell_type": "code", "execution_count": 77, "id": "39ca622b-3e9f-4e0b-bbc4-eff9f21ed63c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "ratings_disabled\n", "False 0.991181\n", "True 0.008819\n", "Name: proportion, dtype: float64" ] }, "execution_count": 77, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['ratings_disabled'].value_counts(normalize=True)" ] }, { "cell_type": "code", "execution_count": 79, "id": "efe7701b-4813-435f-9641-f5e89f4d8c66", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "comments_disabled\n", "False 0.982734\n", "True 0.017266\n", "Name: proportion, dtype: float64" ] }, "execution_count": 79, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['comments_disabled'].value_counts(normalize=True)" ] }, { "cell_type": "code", "execution_count": 103, "id": "8eb3b362-9e0a-4b6b-9c9d-e71c12599500", "metadata": {}, "outputs": [], "source": [ "df = df[\n", " (df['comments_disabled'] == False) &\n", " (df['ratings_disabled'] == False)\n", "].copy()" ] }, { "cell_type": "code", "execution_count": 105, "id": "4a468ebf-0138-4599-86a6-99583f208135", "metadata": {}, "outputs": [], "source": [ "df = df.drop(columns=['ratings_disabled', 'comments_disabled'])\n", "df = df.drop(columns=['time_frame'])\n", "df = df.drop(columns=['index', 'video_error_or_removed'])" ] }, { "cell_type": "code", "execution_count": 107, "id": "6cb877aa-eb15-4f73-848c-fa24c89a704f", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
video_idtrending_datetitlechannel_titlecategory_idpublish_datepublished_day_of_weekpublish_countrytagsviewslikesdislikescomment_counttags_counttags_lengthpublish_hour
02kyS6SvSYSE2017-11-14WE WANT TO TALK ABOUT OUR MARRIAGECaseyNeistat222017-11-13MondayUSSHANtell martin7483745752729661595411517
11ZAPwfrtAFY2017-11-14The Trump Presidency: Last Week Tonight with J...LastWeekTonight242017-11-13MondayUSlast week tonight trump presidency|\"last week ...24187839718561461270341087
25qpjK5DgCt42017-11-14Racist Superman | Rudy Mancuso, King Bach & Le...Rudy Mancuso232017-11-12SundayUSracist superman|\"rudy\"\"|\"\"mancuso\"\"|\"\"king\"\"|\"...3191434146033533981812335619
3puqaWrEC7tY2017-11-14Nickelback Lyrics: Real or Fake?Good Mythical Morning242017-11-13MondayUSrhett and link|\"gmm\"\"|\"\"good mythical morning\"...3431681017266621462752911
4d380meD0W0M2017-11-14I Dare You: GOING BALD!?nigahiga242017-11-12SundayUSryan|\"higa\"\"|\"\"higatv\"\"|\"\"nigahiga\"\"|\"\"i dare ...20957311322351989175181414618
\n", "
" ], "text/plain": [ " video_id trending_date \\\n", "0 2kyS6SvSYSE 2017-11-14 \n", "1 1ZAPwfrtAFY 2017-11-14 \n", "2 5qpjK5DgCt4 2017-11-14 \n", "3 puqaWrEC7tY 2017-11-14 \n", "4 d380meD0W0M 2017-11-14 \n", "\n", " title channel_title \\\n", "0 WE WANT TO TALK ABOUT OUR MARRIAGE CaseyNeistat \n", "1 The Trump Presidency: Last Week Tonight with J... LastWeekTonight \n", "2 Racist Superman | Rudy Mancuso, King Bach & Le... Rudy Mancuso \n", "3 Nickelback Lyrics: Real or Fake? Good Mythical Morning \n", "4 I Dare You: GOING BALD!? nigahiga \n", "\n", " category_id publish_date published_day_of_week publish_country \\\n", "0 22 2017-11-13 Monday US \n", "1 24 2017-11-13 Monday US \n", "2 23 2017-11-12 Sunday US \n", "3 24 2017-11-13 Monday US \n", "4 24 2017-11-12 Sunday US \n", "\n", " tags views likes \\\n", "0 SHANtell martin 748374 57527 \n", "1 last week tonight trump presidency|\"last week ... 2418783 97185 \n", "2 racist superman|\"rudy\"\"|\"\"mancuso\"\"|\"\"king\"\"|\"... 3191434 146033 \n", "3 rhett and link|\"gmm\"\"|\"\"good mythical morning\"... 343168 10172 \n", "4 ryan|\"higa\"\"|\"\"higatv\"\"|\"\"nigahiga\"\"|\"\"i dare ... 2095731 132235 \n", "\n", " dislikes comment_count tags_count tags_length publish_hour \n", "0 2966 15954 1 15 17 \n", "1 6146 12703 4 108 7 \n", "2 5339 8181 23 356 19 \n", "3 666 2146 27 529 11 \n", "4 1989 17518 14 146 18 " ] }, "execution_count": 107, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": 109, "id": "a9ca71b1-bb60-40d5-8331-6a750bea3a6f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Index(['video_id', 'trending_date', 'title', 'channel_title', 'category_id',\n", " 'publish_date', 'published_day_of_week', 'publish_country', 'tags',\n", " 'views', 'likes', 'dislikes', 'comment_count', 'tags_count',\n", " 'tags_length', 'publish_hour'],\n", " dtype='object')" ] }, "execution_count": 109, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.columns" ] }, { "cell_type": "code", "execution_count": 113, "id": "68e096a4-0f1e-4ec3-8430-fa8e2cd24060", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "count 1.580340e+05\n", "mean 7.185344e+03\n", "std 3.439382e+04\n", "min 0.000000e+00\n", "25% 3.080000e+02\n", "50% 1.198000e+03\n", "75% 4.275000e+03\n", "max 1.626501e+06\n", "Name: comment_count, dtype: float64" ] }, "execution_count": 113, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['comment_count'].describe()" ] }, { "cell_type": "code", "execution_count": 115, "id": "263fb98b-38c6-48b5-8ca7-4c9ff30a8dec", "metadata": {}, "outputs": [], "source": [ "df['comment_bin'] = pd.qcut(\n", " df['comment_count'],\n", " q=5,\n", " labels=False,\n", " duplicates='drop'\n", ")" ] }, { "cell_type": "code", "execution_count": 117, "id": "a106d7fe-86b2-43d9-9ee9-14970ad1357b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "comment_bin\n", "0 31631\n", "3 31606\n", "4 31604\n", "2 31597\n", "1 31596\n", "Name: count, dtype: int64" ] }, "execution_count": 117, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['comment_bin'].value_counts()" ] }, { "cell_type": "code", "execution_count": 119, "id": "3904de59-93bb-49ad-80ad-2693ff26ddf1", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/var/folders/t0/wrlt9dys2m59fs2jsn0_bb280000gn/T/ipykernel_2379/1056370390.py:3: FutureWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n", " .apply(lambda x: x.sample(n=10000, random_state=42))\n" ] } ], "source": [ "balanced_df = (\n", " df.groupby('comment_bin', group_keys=False)\n", " .apply(lambda x: x.sample(n=10000, random_state=42))\n", " .copy()\n", ")" ] }, { "cell_type": "code", "execution_count": 121, "id": "59da8775-3199-4f47-9a51-896b1cb78e70", "metadata": {}, "outputs": [], "source": [ "balanced_df.drop(columns=['comment_bin'], inplace=True)" ] }, { "cell_type": "code", "execution_count": 123, "id": "d72c41b6-abbf-4c11-a02a-0a1a148da89e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(50000, 16)" ] }, "execution_count": 123, "metadata": {}, "output_type": "execute_result" } ], "source": [ "balanced_df.shape" ] }, { "cell_type": "code", "execution_count": 125, "id": "9091f98c-0352-48ee-96be-5bc82aa94a81", "metadata": {}, "outputs": [], "source": [ "balanced_df['log_comments'] = np.log1p(balanced_df['comment_count'])" ] }, { "cell_type": "code", "execution_count": 159, "id": "d3e73f6f-c665-4f58-adf7-30c9f312b540", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
video_idtrending_datetitlechannel_titlecategory_idpublish_datepublished_day_of_weekpublish_countrytagsviews...dislikescomment_counttags_counttags_lengthpublish_hourlog_commentspublish_monthpublish_yearis_weekenddays_to_trend
1055487z_dSRE6KAg2018-03-24Mohammed VI, gravement malade, prépare son fi...365 Days252018-03-15ThursdayFRANCEMohammed VI\"|\"Maroc\"|\"News Maroc\"|\"fils\"|\"la s...46923...66163775165.09986632018False9.0
10182820xQIUzWQsY2018-03-06Camille Combal parodie Agathe Auproux sur Inst...Touche pas à mon poste !242018-03-05MondayFRANCETPMP\"|\"Cyril Hanouna\"|\"Touche pas à mon poste...74255...611168141104.76217432018False1.0
68985w3Rd-KZeVp42018-04-06Khabib Nurmagomedov Comments on Confrontation ...MMAWeekly.com172018-04-05ThursdayGBMMA|\"mixed martial arts\"|\"MMAWeekly.com\"|\"MMAW...64009...1121051117904.66343942018False1.0
44121KfD8nLAeMDQ2017-11-29Best Friend From Heaven - Trailer (2017)Brain Power Studio12017-11-16ThursdayGBfamily film|\"movie\"|\"dog\"|\"romance movie\"|\"kid...83872...79759127204.330733112017False13.0
31046FhhlMSpvCQE2018-04-26Haim Gets Ready for Coachella | VogueVogue262018-04-20FridayUScoachella|\"haim\"\"|\"\"haim coachella\"\"|\"\"coachel...169972...14614033560134.94876042018False6.0
\n", "

5 rows × 21 columns

\n", "
" ], "text/plain": [ " video_id trending_date \\\n", "105548 7z_dSRE6KAg 2018-03-24 \n", "101828 20xQIUzWQsY 2018-03-06 \n", "68985 w3Rd-KZeVp4 2018-04-06 \n", "44121 KfD8nLAeMDQ 2017-11-29 \n", "31046 FhhlMSpvCQE 2018-04-26 \n", "\n", " title \\\n", "105548 Mohammed VI, gravement malade, prépare son fi... \n", "101828 Camille Combal parodie Agathe Auproux sur Inst... \n", "68985 Khabib Nurmagomedov Comments on Confrontation ... \n", "44121 Best Friend From Heaven - Trailer (2017) \n", "31046 Haim Gets Ready for Coachella | Vogue \n", "\n", " channel_title category_id publish_date \\\n", "105548 365 Days 25 2018-03-15 \n", "101828 Touche pas à mon poste ! 24 2018-03-05 \n", "68985 MMAWeekly.com 17 2018-04-05 \n", "44121 Brain Power Studio 1 2017-11-16 \n", "31046 Vogue 26 2018-04-20 \n", "\n", " published_day_of_week publish_country \\\n", "105548 Thursday FRANCE \n", "101828 Monday FRANCE \n", "68985 Thursday GB \n", "44121 Thursday GB \n", "31046 Friday US \n", "\n", " tags views ... \\\n", "105548 Mohammed VI\"|\"Maroc\"|\"News Maroc\"|\"fils\"|\"la s... 46923 ... \n", "101828 TPMP\"|\"Cyril Hanouna\"|\"Touche pas à mon poste... 74255 ... \n", "68985 MMA|\"mixed martial arts\"|\"MMAWeekly.com\"|\"MMAW... 64009 ... \n", "44121 family film|\"movie\"|\"dog\"|\"romance movie\"|\"kid... 83872 ... \n", "31046 coachella|\"haim\"\"|\"\"haim coachella\"\"|\"\"coachel... 169972 ... \n", "\n", " dislikes comment_count tags_count tags_length publish_hour \\\n", "105548 66 163 7 75 16 \n", "101828 61 116 8 141 10 \n", "68985 112 105 11 179 0 \n", "44121 79 75 9 127 20 \n", "31046 146 140 33 560 13 \n", "\n", " log_comments publish_month publish_year is_weekend days_to_trend \n", "105548 5.099866 3 2018 False 9.0 \n", "101828 4.762174 3 2018 False 1.0 \n", "68985 4.663439 4 2018 False 1.0 \n", "44121 4.330733 11 2017 False 13.0 \n", "31046 4.948760 4 2018 False 6.0 \n", "\n", "[5 rows x 21 columns]" ] }, "execution_count": 159, "metadata": {}, "output_type": "execute_result" } ], "source": [ "balanced_df.head()" ] }, { "cell_type": "code", "execution_count": 157, "id": "838fcbf8-826f-41d6-8b1c-42b1b7915c2f", "metadata": {}, "outputs": [], "source": [ "balanced_df['publish_month'] = balanced_df['publish_date'].dt.month\n", "balanced_df['publish_year'] = balanced_df['publish_date'].dt.year\n", "balanced_df['is_weekend'] = balanced_df['publish_date'].dt.weekday >= 5\n", "balanced_df['days_to_trend'] = (\n", " balanced_df['trending_date'] - df['publish_date']\n", ").dt.days" ] }, { "cell_type": "code", "execution_count": 161, "id": "2bf812be-4225-49af-8a91-e4725cf4b172", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Index(['video_id', 'trending_date', 'title', 'channel_title', 'category_id',\n", " 'publish_date', 'published_day_of_week', 'publish_country', 'tags',\n", " 'views', 'likes', 'dislikes', 'comment_count', 'tags_count',\n", " 'tags_length', 'publish_hour', 'log_comments', 'publish_month',\n", " 'publish_year', 'is_weekend', 'days_to_trend'],\n", " dtype='object')" ] }, "execution_count": 161, "metadata": {}, "output_type": "execute_result" } ], "source": [ "balanced_df.columns" ] }, { "cell_type": "code", "execution_count": 165, "id": "366399c8-fe00-4715-b587-91fe4e3a6c70", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "4" ] }, "execution_count": 165, "metadata": {}, "output_type": "execute_result" } ], "source": [ "balanced_df['publish_country'].nunique()" ] }, { "cell_type": "code", "execution_count": 311, "id": "b668239a-369d-4912-852f-cf5163c8d944", "metadata": {}, "outputs": [], "source": [ "import re\n", "\n", "STOPWORDS = {\n", " \"the\",\"a\",\"an\",\"and\",\"or\",\"to\",\"of\",\"in\",\"on\",\"for\",\"with\",\"at\",\"by\",\n", " \"official\",\"video\",\"trailer\",\"full\",\"episode\",\"new\",\"music\",\"mv\",\"vs\",\n", " \"best\",\"from\",\"ready\",\"live\",\"feat\",\"ft\",\"hd\",\"4k\",\"2024\",\"2023\",\n", " \"season\",\"part\",\"game\",\"clip\",\"reaction\"\n", "}\n", "\n", "def make_keyword(title: str, max_words: int = 3) -> str:\n", " if not isinstance(title, str):\n", " return \"\"\n", " t = title.lower()\n", " t = re.sub(r\"http\\S+|www\\.\\S+\", \" \", t)\n", " t = re.sub(r\"[^a-z0-9\\s]\", \" \", t)\n", " words = [w for w in t.split() if w not in STOPWORDS and len(w) > 2 and not w.isdigit()]\n", " return \" \".join(words[:max_words])\n" ] }, { "cell_type": "code", "execution_count": 313, "id": "b1318d08-4199-4801-a620-5fad7f9da312", "metadata": {}, "outputs": [], "source": [ "balanced_df[\"keyword\"] = balanced_df[\"title\"].apply(make_keyword)" ] }, { "cell_type": "code", "execution_count": 315, "id": "cb36fcc2-f730-4325-b5c2-d8be75077333", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
video_idtrending_datetitlechannel_titlecategory_idpublish_datepublished_day_of_weekpublish_countrytagsviews...tags_counttags_lengthpublish_hourlog_commentspublish_monthpublish_yearis_weekenddays_to_trendkeywordtrend_score
1055487z_dSRE6KAg2018-03-24Mohammed VI, gravement malade, prépare son fi...365 Days252018-03-15ThursdayFRANCEMohammed VI\"|\"Maroc\"|\"News Maroc\"|\"fils\"|\"la s...46923...775165.09986632018False9.0mohammed gravement maladeNaN
10182820xQIUzWQsY2018-03-06Camille Combal parodie Agathe Auproux sur Inst...Touche pas à mon poste !242018-03-05MondayFRANCETPMP\"|\"Cyril Hanouna\"|\"Touche pas à mon poste...74255...8141104.76217432018False1.0camille combal parodieNaN
68985w3Rd-KZeVp42018-04-06Khabib Nurmagomedov Comments on Confrontation ...MMAWeekly.com172018-04-05ThursdayGBMMA|\"mixed martial arts\"|\"MMAWeekly.com\"|\"MMAW...64009...1117904.66343942018False1.0khabib nurmagomedov commentsNaN
44121KfD8nLAeMDQ2017-11-29Best Friend From Heaven - Trailer (2017)Brain Power Studio12017-11-16ThursdayGBfamily film|\"movie\"|\"dog\"|\"romance movie\"|\"kid...83872...9127204.330733112017False13.0friend heavenNaN
31046FhhlMSpvCQE2018-04-26Haim Gets Ready for Coachella | VogueVogue262018-04-20FridayUScoachella|\"haim\"\"|\"\"haim coachella\"\"|\"\"coachel...169972...33560134.94876042018False6.0haim gets coachellaNaN
\n", "

5 rows × 23 columns

\n", "
" ], "text/plain": [ " video_id trending_date \\\n", "105548 7z_dSRE6KAg 2018-03-24 \n", "101828 20xQIUzWQsY 2018-03-06 \n", "68985 w3Rd-KZeVp4 2018-04-06 \n", "44121 KfD8nLAeMDQ 2017-11-29 \n", "31046 FhhlMSpvCQE 2018-04-26 \n", "\n", " title \\\n", "105548 Mohammed VI, gravement malade, prépare son fi... \n", "101828 Camille Combal parodie Agathe Auproux sur Inst... \n", "68985 Khabib Nurmagomedov Comments on Confrontation ... \n", "44121 Best Friend From Heaven - Trailer (2017) \n", "31046 Haim Gets Ready for Coachella | Vogue \n", "\n", " channel_title category_id publish_date \\\n", "105548 365 Days 25 2018-03-15 \n", "101828 Touche pas à mon poste ! 24 2018-03-05 \n", "68985 MMAWeekly.com 17 2018-04-05 \n", "44121 Brain Power Studio 1 2017-11-16 \n", "31046 Vogue 26 2018-04-20 \n", "\n", " published_day_of_week publish_country \\\n", "105548 Thursday FRANCE \n", "101828 Monday FRANCE \n", "68985 Thursday GB \n", "44121 Thursday GB \n", "31046 Friday US \n", "\n", " tags views ... \\\n", "105548 Mohammed VI\"|\"Maroc\"|\"News Maroc\"|\"fils\"|\"la s... 46923 ... \n", "101828 TPMP\"|\"Cyril Hanouna\"|\"Touche pas à mon poste... 74255 ... \n", "68985 MMA|\"mixed martial arts\"|\"MMAWeekly.com\"|\"MMAW... 64009 ... \n", "44121 family film|\"movie\"|\"dog\"|\"romance movie\"|\"kid... 83872 ... \n", "31046 coachella|\"haim\"\"|\"\"haim coachella\"\"|\"\"coachel... 169972 ... \n", "\n", " tags_count tags_length publish_hour log_comments publish_month \\\n", "105548 7 75 16 5.099866 3 \n", "101828 8 141 10 4.762174 3 \n", "68985 11 179 0 4.663439 4 \n", "44121 9 127 20 4.330733 11 \n", "31046 33 560 13 4.948760 4 \n", "\n", " publish_year is_weekend days_to_trend keyword \\\n", "105548 2018 False 9.0 mohammed gravement malade \n", "101828 2018 False 1.0 camille combal parodie \n", "68985 2018 False 1.0 khabib nurmagomedov comments \n", "44121 2017 False 13.0 friend heaven \n", "31046 2018 False 6.0 haim gets coachella \n", "\n", " trend_score \n", "105548 NaN \n", "101828 NaN \n", "68985 NaN \n", "44121 NaN \n", "31046 NaN \n", "\n", "[5 rows x 23 columns]" ] }, "execution_count": 315, "metadata": {}, "output_type": "execute_result" } ], "source": [ "balanced_df.head()" ] }, { "cell_type": "code", "execution_count": 431, "id": "edcf1068-df70-4c5a-8258-c2d8f43a93ba", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "399" ] }, "execution_count": 431, "metadata": {}, "output_type": "execute_result" } ], "source": [ "kw = (balanced_df[\"keyword\"]\n", " .value_counts()\n", " .head(400)\n", " .index\n", " .tolist())\n", "kw = [k for k in kw if k]\n", "len(kw)\n" ] }, { "cell_type": "markdown", "id": "61777518-257c-44eb-8755-b43713f0bd14", "metadata": {}, "source": [ "## Adding a Google Trends Feature" ] }, { "cell_type": "code", "execution_count": 433, "id": "b69e7e1e-0bd5-40ff-87f2-965cfe67a9d0", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n", "/Users/sanzhar/ML_project/ner_env/lib/python3.11/site-packages/pytrends/request.py:260: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " df = df.fillna(False)\n" ] }, { "data": { "text/plain": [ "(399,\n", " [('star wars last', 0.0),\n", " ('avengers infinity war', 0.0),\n", " ('vaani rani', 0.0),\n", " ('marvel studios avengers', 0.0),\n", " ('crime patrol dial', 0.0)])" ] }, "execution_count": 433, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import time\n", "import pandas as pd\n", "from pytrends.request import TrendReq\n", "\n", "def fetch_trends_scores(keywords, geo=\"\", timeframe=\"2017-01-01 2017-12-31\", sleep_s=2):\n", " pytrends = TrendReq(hl=\"en-US\", tz=360)\n", " out = {}\n", "\n", " for i in range(0, len(keywords), 5):\n", " batch = keywords[i:i+5]\n", " try:\n", " pytrends.build_payload(batch, timeframe=timeframe, geo=geo)\n", " it = pytrends.interest_over_time()\n", "\n", " if it is None or it.empty:\n", " for k in batch:\n", " out[k] = 0.0\n", " else:\n", " for k in batch:\n", " out[k] = float(it[k].mean()) if k in it.columns else 0.0\n", "\n", " except Exception:\n", " for k in batch:\n", " out[k] = 0.0\n", "\n", " time.sleep(sleep_s)\n", "\n", " return out\n", "\n", "trend_map = fetch_trends_scores(kw, geo=\"\", timeframe=\"today 1-m\", sleep_s=2)\n", "len(trend_map), list(trend_map.items())[:5]\n" ] }, { "cell_type": "code", "execution_count": 435, "id": "c154f25d-8ccc-4a89-b78f-854896d86309", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
keywordtrend_score
105548mohammed gravement malade0.0
101828camille combal parodie0.0
68985khabib nurmagomedov comments0.0
44121friend heaven0.0
31046haim gets coachella0.0
42054sweetie destroys flav0.0
54853answer not hitchhiker0.0
14666vision xfl0.0
834320.0
133498louis domingue steve0.0
\n", "
" ], "text/plain": [ " keyword trend_score\n", "105548 mohammed gravement malade 0.0\n", "101828 camille combal parodie 0.0\n", "68985 khabib nurmagomedov comments 0.0\n", "44121 friend heaven 0.0\n", "31046 haim gets coachella 0.0\n", "42054 sweetie destroys flav 0.0\n", "54853 answer not hitchhiker 0.0\n", "14666 vision xfl 0.0\n", "83432 0.0\n", "133498 louis domingue steve 0.0" ] }, "execution_count": 435, "metadata": {}, "output_type": "execute_result" } ], "source": [ "balanced_df[\"trend_score\"] = balanced_df[\"keyword\"].map(trend_map).fillna(0.0)\n", "balanced_df[[\"keyword\", \"trend_score\"]].head(10)" ] }, { "cell_type": "code", "execution_count": 437, "id": "63aef714-eeef-46d6-9054-8839b3f3af8a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "105548 False\n", "101828 False\n", "68985 False\n", "44121 False\n", "31046 False\n", " ... \n", "71508 False\n", "43386 False\n", "41541 False\n", "23069 False\n", "19404 False\n", "Name: trend_score, Length: 50000, dtype: bool" ] }, "execution_count": 437, "metadata": {}, "output_type": "execute_result" } ], "source": [ "balanced_df.trend_score.isnull()" ] }, { "cell_type": "code", "execution_count": 439, "id": "34afda02-2d59-4300-acd7-fb30184634ff", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "21320" ] }, "execution_count": 439, "metadata": {}, "output_type": "execute_result" } ], "source": [ "balanced_df[\"keyword\"].nunique()" ] }, { "cell_type": "code", "execution_count": 443, "id": "b853d488-5744-45e8-b219-f213ad17f012", "metadata": {}, "outputs": [], "source": [ "features = [\n", " 'tags_count',\n", " 'tags_length',\n", " 'publish_hour',\n", " 'publish_month',\n", " 'days_to_trend',\n", " 'is_weekend',\n", " 'category_id',\n", " 'publish_country',\n", " 'trend_score'\n", "]\n", "\n", "X = balanced_df[features]\n", "y = balanced_df['log_comments']\n" ] }, { "cell_type": "code", "execution_count": 445, "id": "9428cdcb-0068-4a8b-b06f-ca3d323f2f0b", "metadata": {}, "outputs": [], "source": [ "X = pd.get_dummies(\n", " X,\n", " columns=['publish_country'],\n", " drop_first=True\n", ")\n" ] }, { "cell_type": "code", "execution_count": 447, "id": "c0a821e2-3dc9-482e-9795-507d97a42d6b", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "Index(['tags_count', 'tags_length', 'publish_hour', 'publish_month',\n", " 'days_to_trend', 'is_weekend', 'category_id', 'trend_score',\n", " 'publish_country_FRANCE', 'publish_country_GB', 'publish_country_US'],\n", " dtype='object')" ] }, "execution_count": 447, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X.columns" ] }, { "cell_type": "code", "execution_count": 449, "id": "da99e147-c73d-42e3-ba70-99cd7ff5129c", "metadata": {}, "outputs": [], "source": [ "X_train, X_test, y_train, y_test = train_test_split(\n", " X, y,\n", " test_size=0.2,\n", " random_state=42\n", ")" ] }, { "cell_type": "code", "execution_count": 451, "id": "b105cfa7-cda7-4631-8c78-ba6e7772b3e1", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "((40000, 11), (10000, 11))" ] }, "execution_count": 451, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X_train.shape, X_test.shape" ] }, { "cell_type": "code", "execution_count": 453, "id": "ac5b064d-c5f5-4b3e-8f42-3dd0a8ef6b0f", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
LinearRegression()
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "LinearRegression()" ] }, "execution_count": 453, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model = LinearRegression()\n", "model.fit(X_train, y_train)" ] }, { "cell_type": "code", "execution_count": 455, "id": "5bfb5975-f2e2-497d-946e-5c7ce211dc85", "metadata": {}, "outputs": [], "source": [ "preds = model.predict(X_test)" ] }, { "cell_type": "code", "execution_count": 457, "id": "510e10a9-31cb-44ab-bfb8-374ddaf5f3e5", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "RMSE: 1.8412141156327049\n", "R2: 0.23028675785045494\n" ] } ], "source": [ "print(\"RMSE:\", np.sqrt(mean_squared_error(y_test, preds)))\n", "print(\"R2:\", r2_score(y_test, preds))" ] }, { "cell_type": "code", "execution_count": 459, "id": "f82ff8e3-6cde-4660-95a4-19b60406bc0f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "RF RMSE: 1.3016674515473314\n", "RF R2: 0.6153015314103947\n" ] } ], "source": [ "rf = RandomForestRegressor(n_estimators=100, random_state=42)\n", "rf.fit(X_train, y_train)\n", "\n", "preds_rf = rf.predict(X_test)\n", "\n", "print(\"RF RMSE:\", np.sqrt(mean_squared_error(y_test, preds_rf)))\n", "print(\"RF R2:\", r2_score(y_test, preds_rf))" ] }, { "cell_type": "code", "execution_count": 460, "id": "e3a21067-eefc-4dad-a1c9-b57233aa2862", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tags_length 0.195252\n", "publish_country_FRANCE 0.179267\n", "publish_hour 0.134345\n", "tags_count 0.130771\n", "category_id 0.110313\n", "days_to_trend 0.109666\n", "publish_month 0.077705\n", "is_weekend 0.022079\n", "trend_score 0.019728\n", "publish_country_US 0.011847\n", "publish_country_GB 0.009027\n", "dtype: float64\n" ] } ], "source": [ "importances = pd.Series(\n", " rf.feature_importances_,\n", " index=X.columns\n", ").sort_values(ascending=False)\n", "\n", "print(importances.head(11))\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f814eaee-4256-428f-89b9-035cf0024ad6", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3.11 (NER)", "language": "python", "name": "ner_env" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.14" } }, "nbformat": 4, "nbformat_minor": 5 }