# importing GraphLab
import graphlab

# dataset in CSV format https://d396qusza40orc.cloudfront.net/phoenixassets/amazon_baby.csv
products = graphlab.SFrame('coursera-notebooks/course-1/amazon_baby.gl')

[INFO] This non-commercial license of GraphLab Create is assigned to prashantgonarkar@gmail.com and will expire on February 13, 2017. For commercial licensing options, visit https://dato.com/buy/.

[INFO] Start server at: ipc:///tmp/graphlab_server-1172 - Server binary: /usr/local/lib/python2.7/dist-packages/graphlab/unity_server - Server log: /tmp/graphlab_server_1455978510.log
[INFO] GraphLab Server Version: 1.8

products.head()

products['name' == 'Planetwise Flannel Wipes']

{'name': 'Planetwise Flannel Wipes',
 'rating': 3.0,
 'review': 'These flannel wipes are OK, but in my opinion not worth keeping.  I also ordered someImse Vimse Cloth Wipes-Ocean Blue-12 countwhich are larger, had a nicer, softer texture and just seemed higher quality.  I use cloth wipes for hands and faces and have been usingThirsties 6 Pack Fab Wipes, Boyfor about 8 months now and need to replace them because they are starting to get rough and have had stink issues for a while that stripping no longer handles.'}

products['word_count'] = graphlab.text_analytics.count_words(products['review'])

products.head()

products['name' == 'Planetwise Flannel Wipes']

{'name': 'Planetwise Flannel Wipes',
 'rating': 3.0,
 'review': 'These flannel wipes are OK, but in my opinion not worth keeping.  I also ordered someImse Vimse Cloth Wipes-Ocean Blue-12 countwhich are larger, had a nicer, softer texture and just seemed higher quality.  I use cloth wipes for hands and faces and have been usingThirsties 6 Pack Fab Wipes, Boyfor about 8 months now and need to replace them because they are starting to get rough and have had stink issues for a while that stripping no longer handles.',
 'word_count': {'6': 1,
  '8': 1,
  'a': 2,
  'about': 1,
  'also': 1,
  'and': 5,
  'are': 3,
  'because': 1,
  'been': 1,
  'blue-12': 1,
  'boyfor': 1,
  'but': 1,
  'cloth': 2,
  'countwhich': 1,
  'fab': 1,
  'faces': 1,
  'flannel': 1,
  'for': 2,
  'get': 1,
  'had': 2,
  'handles.': 1,
  'hands': 1,
  'have': 2,
  'higher': 1,
  'i': 2,
  'in': 1,
  'issues': 1,
  'just': 1,
  'keeping.': 1,
  'larger,': 1,
  'longer': 1,
  'months': 1,
  'my': 1,
  'need': 1,
  'nicer,': 1,
  'no': 1,
  'not': 1,
  'now': 1,
  'ok,': 1,
  'opinion': 1,
  'ordered': 1,
  'pack': 1,
  'quality.': 1,
  'replace': 1,
  'rough': 1,
  'seemed': 1,
  'softer': 1,
  'someimse': 1,
  'starting': 1,
  'stink': 1,
  'stripping': 1,
  'texture': 1,
  'that': 1,
  'them': 1,
  'these': 1,
  'they': 1,
  'to': 2,
  'use': 1,
  'usingthirsties': 1,
  'vimse': 1,
  'while': 1,
  'wipes': 2,
  'wipes,': 1,
  'wipes-ocean': 1,
  'worth': 1}}

graphlab.canvas.set_target('ipynb')

products['name'].show()

# Explore Vulli Sophie

giraffe_reviews = products[products['name'] == 'Vulli Sophie the Giraffe Teether']

len(giraffe_reviews)

785

giraffe_reviews['rating'].show(view='Categorical')

products['rating'].show(view='Categorical')

#ignore all 3 reviews 
products = products[products['rating'] != 3 ]

len(products)

166752

# product sentiment = 4* or 5 * reviews 
products['sentiment'] = products['rating'] >=4

products.head()

train_data,test_data = products.random_split(0.8,seed=0)

sentiment_model = graphlab.logistic_classifier.create(train_data,
                                                     target='sentiment',
                                                     features=['word_count'],
                                                     validation_set=test_data)

PROGRESS: Logistic regression:
PROGRESS: --------------------------------------------------------
PROGRESS: Number of examples          : 133448
PROGRESS: Number of classes           : 2
PROGRESS: Number of feature columns   : 1
PROGRESS: Number of unpacked features : 219217
PROGRESS: Number of coefficients    : 219218
PROGRESS: Starting L-BFGS
PROGRESS: --------------------------------------------------------
PROGRESS: +-----------+----------+-----------+--------------+-------------------+---------------------+
PROGRESS: | Iteration | Passes   | Step size | Elapsed Time | Training-accuracy | Validation-accuracy |
PROGRESS: +-----------+----------+-----------+--------------+-------------------+---------------------+
PROGRESS: | 1         | 5        | 0.000002  | 3.822951     | 0.841481          | 0.839989            |
PROGRESS: | 2         | 9        | 3.000000  | 6.524749     | 0.947425          | 0.894877            |
PROGRESS: | 3         | 10       | 3.000000  | 7.537581     | 0.923768          | 0.866232            |
PROGRESS: | 4         | 11       | 3.000000  | 8.551528     | 0.971779          | 0.912743            |
PROGRESS: | 5         | 12       | 3.000000  | 9.571802     | 0.975511          | 0.908900            |
PROGRESS: | 6         | 13       | 3.000000  | 10.587506    | 0.899991          | 0.825967            |
PROGRESS: | 7         | 15       | 1.000000  | 12.165112    | 0.984548          | 0.921451            |
PROGRESS: | 8         | 16       | 1.000000  | 13.183553    | 0.985118          | 0.921871            |
PROGRESS: | 9         | 17       | 1.000000  | 14.204816    | 0.987066          | 0.919709            |
PROGRESS: | 10        | 18       | 1.000000  | 15.215279    | 0.988715          | 0.916256            |
PROGRESS: +-----------+----------+-----------+--------------+-------------------+---------------------+
PROGRESS: TERMINATED: Iteration limit reached.
PROGRESS: This model may not be optimal. To improve it, consider increasing `max_iterations`.

sentiment_model.evaluate(test_data,metric='roc_curve')

{'roc_curve': Columns:
    threshold   float
    fpr float
    tpr float
    p   int
    n   int

 Rows: 100001

 Data:
 +-----------+----------------+----------------+-------+------+
 | threshold |      fpr       |      tpr       |   p   |  n   |
 +-----------+----------------+----------------+-------+------+
 |    0.0    |      1.0       |      1.0       | 27976 | 5328 |
 |   1e-05   | 0.909346846847 | 0.998856162425 | 27976 | 5328 |
 |   2e-05   | 0.896021021021 | 0.998748927652 | 27976 | 5328 |
 |   3e-05   | 0.886448948949 | 0.998462968259 | 27976 | 5328 |
 |   4e-05   | 0.879692192192 | 0.998284243637 | 27976 | 5328 |
 |   5e-05   | 0.875187687688 | 0.998212753789 | 27976 | 5328 |
 |   6e-05   | 0.872184684685 | 0.998177008865 | 27976 | 5328 |
 |   7e-05   | 0.868618618619 | 0.998034029168 | 27976 | 5328 |
 |   8e-05   | 0.864677177177 | 0.997998284244 | 27976 | 5328 |
 |   9e-05   | 0.860735735736 | 0.997962539319 | 27976 | 5328 |
 +-----------+----------------+----------------+-------+------+
 [100001 rows x 5 columns]
 Note: Only the head of the SFrame is printed.
 You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.}

sentiment_model.show(view='Evaluation')

giraffe_reviews['predicted_sentiment'] = sentiment_model.predict(giraffe_reviews,output_type='probability')

giraffe_reviews.head()

giraffe_reviews = giraffe_reviews.sort('predicted_sentiment',ascending=False)

giraffe_reviews.head()

giraffe_reviews[0]['review']

"Sophie, oh Sophie, your time has come. My granddaughter, Violet is 5 months old and starting to teeth. What joy little Sophie brings to Violet. Sophie is made of a very pliable rubber that is sturdy but not tough. It is quite easy for Violet to twist Sophie into unheard of positions to get Sophie into her mouth. The little nose and hooves fit perfectly into small mouths, and the drooling has purpose. The paint on Sophie is food quality.Sophie was born in 1961 in France. The maker had wondered why there was nothing available for babies and made Sophie from the finest rubber, phthalate-free on St Sophie's Day, thus the name was born. Since that time millions of Sophie's populate the world. She is soft and for babies little hands easy to grasp. Violet especially loves the bumpy head and horns of Sophie. Sophie has a long neck that easy to grasp and twist. She has lovely, sizable spots that attract Violet's attention. Sophie has happy little squeaks that bring squeals of delight from Violet. She is able to make Sophie squeak and that brings much joy. Sophie's smooth skin is soothing to Violet's little gums. Sophie is 7 inches tall and is the exact correct size for babies to hold and love.As you well know the first thing babies grasp, goes into their mouths- how wonderful to have a toy that stimulates all of the senses and helps with the issue of teething. Sophie is small enough to fit into any size pocket or bag. Sophie is the perfect find for babies from a few months to a year old. How wonderful to hear the giggles and laughs that emanate from babies who find Sophie irresistible. Viva La Sophie!Highly Recommended.  prisrob 12-11-09"

giraffe_reviews[1]['review']

"I'm not sure why Sophie is such a hit with the little ones, but my 7 month old baby girl is one of her adoring fans.  The rubber is softer and more pleasant to handle, and my daughter has enjoyed chewing on her legs and the nubs on her head even before she started teething.  She also loves the squeak that Sophie makes when you squeeze her.  Not sure what it is but if Sophie is amongst a pile of her other toys, my daughter will more often than not reach for Sophie.  And I have the peace of mind of knowing that only edible and safe paints and materials have been used to make Sophie, as opposed to Bright Starts and other baby toys made in China.  Now that the research is out on phthalates and other toxic substances in baby toys, I think it's more important than ever to find good quality toys that are also safe for our babies to handle and put in their mouths.  Sophie is a must-have for every new mom in my opinion.  Even if your kid is one of the few that can take or leave her, it's worth a try.  Vulli, the makers of Sophie, also make natural rubber teething rings that my daughter loves as well."

giraffe_reviews[-1]['review']

"My son (now 2.5) LOVED his Sophie, and I bought one for every baby shower I've gone to. Now, my daughter (6 months) just today nearly choked on it and I will never give it to her again. Had I not been within hearing range it could have been fatal. The strange sound she was making caught my attention and when I went to her and found the front curved leg shoved well down her throat and her face a purply/blue I panicked. I pulled it out and she vomited all over the carpet before screaming her head off. I can't believe how my opinion of this toy has changed from a must-have to a must-not-use. Please don't disregard any of the choking hazard comments, they are not over exaggerated!"

giraffe_reviews[-2]['review']

"This children's toy is nostalgic and very cute. However, there is a distinct rubber smell and a very odd taste, yes I tried it, that my baby did not enjoy. Also, if it is soiled it is extremely difficult to clean as the rubber is a kind of porus material and does not clean well. The final thing is the squeaking device inside which stopped working after the first couple of days. I returned this item feeling I had overpaid for a toy that was defective and did not meet my expectations. Please do not be swayed by the cute packaging and hype surounding it as I was. One more thing, I was given a full refund from Amazon without any problem."

def wordcount(x):
    if 'and' in x:
        return x['and']
    else:
        return 0

temp_products = products.head()

temp_products['awesome'] = temp_products['word_count'].apply(wordcount)

temp_products

temp_products['awesome'] = temp_products['word_count'].apply(lambda x: x['it'] if 'it' in x else 0)

temp_products

selected_words = ['awesome', 'great', 'fantastic', 'amazing', 'love', 'horrible', 'bad', 'terrible', 'awful', 'wow', 'hate']

for word in selected_words:
    temp_products[word] = temp_products['word_count'].apply(lambda x: x[word] if word in x else 0)

temp_products

for word in selected_words:
    products[word] = products['word_count'].apply(lambda x: x[word] if word in x else 0)

products.head()

len(products)

166752

products['awesome'].sum()

2002

sumlist = []
for word in selected_words:
    temp_var = products[word].sum()
    print word, "occurance",temp_var
    sumlist.append(temp_var)

awesome occurance 2002
great occurance 42420
fantastic occurance 873
amazing occurance 1305
love occurance 40277
horrible occurance 659
bad occurance 3197
terrible occurance 673
awful occurance 345
wow occurance 131
hate occurance 1057

max(sumlist)

42420

min(sumlist)

131

train_data,test_data = products.random_split(.8, seed=0)

selected_words_model = graphlab.logistic_classifier.create(train_data,
                                                     target='sentiment',
                                                     features=selected_words,
                                                     validation_set=test_data)

PROGRESS: Logistic regression:
PROGRESS: --------------------------------------------------------
PROGRESS: Number of examples          : 133448
PROGRESS: Number of classes           : 2
PROGRESS: Number of feature columns   : 11
PROGRESS: Number of unpacked features : 11
PROGRESS: Number of coefficients    : 12
PROGRESS: Starting Newton Method
PROGRESS: --------------------------------------------------------
PROGRESS: +-----------+----------+--------------+-------------------+---------------------+
PROGRESS: | Iteration | Passes   | Elapsed Time | Training-accuracy | Validation-accuracy |
PROGRESS: +-----------+----------+--------------+-------------------+---------------------+
PROGRESS: | 1         | 2        | 0.550229     | 0.844299          | 0.842842            |
PROGRESS: | 2         | 3        | 0.905510     | 0.844186          | 0.842842            |
PROGRESS: | 3         | 4        | 1.255980     | 0.844276          | 0.843142            |
PROGRESS: | 4         | 5        | 1.606155     | 0.844269          | 0.843142            |
PROGRESS: | 5         | 6        | 1.957893     | 0.844269          | 0.843142            |
PROGRESS: | 6         | 7        | 2.308999     | 0.844269          | 0.843142            |
PROGRESS: +-----------+----------+--------------+-------------------+---------------------+
PROGRESS: SUCCESS: Optimal solution found.
PROGRESS:

selected_words_model['coefficients']

selected_words_model['coefficients']['value'].sort()

dtype: float
Rows: 12
[-2.090499984872604, -1.9965180055877998, -1.7646995563132146, -1.4091640627563817, -0.9858273699287564, -0.05414501233324819, 0.883937894897966, 0.8913030903043112, 0.8928024225084774, 1.0580088887848158, 1.3672831522930367, 1.3998983430174745]

selected_words_model.evaluate(test_data)

{'accuracy': 0.8431419649291376,
 'auc': 0.6648096413721418,
 'confusion_matrix': Columns:
    target_label    int
    predicted_label int
    count   int

 Rows: 4

 Data:
 +--------------+-----------------+-------+
 | target_label | predicted_label | count |
 +--------------+-----------------+-------+
 |      0       |        0        |  234  |
 |      1       |        0        |  130  |
 |      0       |        1        |  5094 |
 |      1       |        1        | 27846 |
 +--------------+-----------------+-------+
 [4 rows x 3 columns],
 'f1_score': 0.914242563530107,
 'log_loss': 0.40547471103673566,
 'precision': 0.8453551912568306,
 'recall': 0.9953531598513011,
 'roc_curve': Columns:
    threshold   float
    fpr float
    tpr float
    p   int
    n   int

 Rows: 100001

 Data:
 +-----------+-----+-----+-------+------+
 | threshold | fpr | tpr |   p   |  n   |
 +-----------+-----+-----+-------+------+
 |    0.0    | 1.0 | 1.0 | 27976 | 5328 |
 |   1e-05   | 1.0 | 1.0 | 27976 | 5328 |
 |   2e-05   | 1.0 | 1.0 | 27976 | 5328 |
 |   3e-05   | 1.0 | 1.0 | 27976 | 5328 |
 |   4e-05   | 1.0 | 1.0 | 27976 | 5328 |
 |   5e-05   | 1.0 | 1.0 | 27976 | 5328 |
 |   6e-05   | 1.0 | 1.0 | 27976 | 5328 |
 |   7e-05   | 1.0 | 1.0 | 27976 | 5328 |
 |   8e-05   | 1.0 | 1.0 | 27976 | 5328 |
 |   9e-05   | 1.0 | 1.0 | 27976 | 5328 |
 +-----------+-----+-----+-------+------+
 [100001 rows x 5 columns]
 Note: Only the head of the SFrame is printed.
 You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.}

sentiment_model.evaluate(test_data)

{'accuracy': 0.916256305548883,
 'auc': 0.9446492867438502,
 'confusion_matrix': Columns:
    target_label    int
    predicted_label int
    count   int

 Rows: 4

 Data:
 +--------------+-----------------+-------+
 | target_label | predicted_label | count |
 +--------------+-----------------+-------+
 |      1       |        0        |  1461 |
 |      0       |        1        |  1328 |
 |      0       |        0        |  4000 |
 |      1       |        1        | 26515 |
 +--------------+-----------------+-------+
 [4 rows x 3 columns],
 'f1_score': 0.9500349343413533,
 'log_loss': 0.26106698432422115,
 'precision': 0.9523039902309378,
 'recall': 0.9477766657134686,
 'roc_curve': Columns:
    threshold   float
    fpr float
    tpr float
    p   int
    n   int

 Rows: 100001

 Data:
 +-----------+----------------+----------------+-------+------+
 | threshold |      fpr       |      tpr       |   p   |  n   |
 +-----------+----------------+----------------+-------+------+
 |    0.0    |      1.0       |      1.0       | 27976 | 5328 |
 |   1e-05   | 0.909346846847 | 0.998856162425 | 27976 | 5328 |
 |   2e-05   | 0.896021021021 | 0.998748927652 | 27976 | 5328 |
 |   3e-05   | 0.886448948949 | 0.998462968259 | 27976 | 5328 |
 |   4e-05   | 0.879692192192 | 0.998284243637 | 27976 | 5328 |
 |   5e-05   | 0.875187687688 | 0.998212753789 | 27976 | 5328 |
 |   6e-05   | 0.872184684685 | 0.998177008865 | 27976 | 5328 |
 |   7e-05   | 0.868618618619 | 0.998034029168 | 27976 | 5328 |
 |   8e-05   | 0.864677177177 | 0.997998284244 | 27976 | 5328 |
 |   9e-05   | 0.860735735736 | 0.997962539319 | 27976 | 5328 |
 +-----------+----------------+----------------+-------+------+
 [100001 rows x 5 columns]
 Note: Only the head of the SFrame is printed.
 You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.}

products['rating'].show(view='Categorical')

 diaper_champ_reviews = products[products['name'] == 'Baby Trend Diaper Champ']

len(diaper_champ_reviews)

298

diaper_champ_reviews['rating'].show(view='Categorical')

diaper_champ_reviews['predicted_sentiment'] = sentiment_model.predict(diaper_champ_reviews,output_type='probability')

diaper_champ_reviews = diaper_champ_reviews.sort('predicted_sentiment',ascending=False)

diaper_champ_reviews.head()

diaper_champ_reviews.head()[0]

{'amazing': 0,
 'awesome': 0,
 'awful': 0,
 'bad': 0,
 'fantastic': 0,
 'great': 0,
 'hate': 0,
 'horrible': 0,
 'love': 0,
 'name': 'Baby Trend Diaper Champ',
 'predicted_sentiment': 0.9999999372669541,
 'rating': 5.0,
 'review': 'Baby Luke can turn a clean diaper to a dirty diaper in 3 seconds flat. The diaper champ turns the smelly diaper into "what diaper smell" in less time than that. I hesitated and wondered what I REALLY needed for the nursery. This is one of the best purchases we made. The champ, the baby bjorn, fluerville diaper bag, and graco pack and play bassinet all vie for the best baby purchase.Great product, easy to use, economical, effective, absolutly fabulous.UpdateI knew that I loved the champ, and useing the diaper genie at a friend\'s house REALLY reinforced that!! There is no comparison, the chanp is easy and smell free, the genie was difficult to use one handed (which is absolutly vital if you have a little one on a changing pad) and there was a deffinite odor eminating from the genieplus we found that the quick tie garbage bags where the ties are integrated into the bag work really well because there isn\'t any added bulk around the sealing edge of the champ.',
 'sentiment': 1,
 'terrible': 0,
 'word_count': {'"what': 1,
  '(which': 1,
  '3': 1,
  'a': 6,
  'absolutly': 2,
  'added': 1,
  'all': 1,
  'and': 6,
  'any': 1,
  'are': 1,
  'around': 1,
  'at': 1,
  'baby': 3,
  'bag': 1,
  'bag,': 1,
  'bags': 1,
  'bassinet': 1,
  'because': 1,
  'best': 2,
  'bjorn,': 1,
  'bulk': 1,
  'can': 1,
  'champ': 1,
  'champ,': 2,
  'champ.': 1,
  'changing': 1,
  'chanp': 1,
  'clean': 1,
  'comparison,': 1,
  'deffinite': 1,
  'diaper': 7,
  'difficult': 1,
  'dirty': 1,
  'easy': 2,
  'economical,': 1,
  'edge': 1,
  'effective,': 1,
  'eminating': 1,
  'fabulous.updatei': 1,
  'flat.': 1,
  'fluerville': 1,
  'for': 2,
  'found': 1,
  'free,': 1,
  "friend's": 1,
  'from': 1,
  'garbage': 1,
  'genie': 2,
  'genieplus': 1,
  'graco': 1,
  'handed': 1,
  'have': 1,
  'hesitated': 1,
  'house': 1,
  'i': 3,
  'if': 1,
  'in': 2,
  'integrated': 1,
  'into': 2,
  'is': 4,
  "isn't": 1,
  'knew': 1,
  'less': 1,
  'little': 1,
  'loved': 1,
  'luke': 1,
  'made.': 1,
  'needed': 1,
  'no': 1,
  'nursery.': 1,
  'odor': 1,
  'of': 2,
  'on': 1,
  'one': 3,
  'pack': 1,
  'pad)': 1,
  'play': 1,
  'product,': 1,
  'purchase.great': 1,
  'purchases': 1,
  'quick': 1,
  'really': 3,
  'reinforced': 1,
  'sealing': 1,
  'seconds': 1,
  'smell': 1,
  'smell"': 1,
  'smelly': 1,
  'than': 1,
  'that': 2,
  'that!!': 1,
  'that.': 1,
  'the': 17,
  'there': 3,
  'this': 1,
  'tie': 1,
  'ties': 1,
  'time': 1,
  'to': 3,
  'turn': 1,
  'turns': 1,
  'use': 1,
  'use,': 1,
  'useing': 1,
  'vie': 1,
  'vital': 1,
  'was': 2,
  'we': 2,
  'well': 1,
  'what': 1,
  'where': 1,
  'wondered': 1,
  'work': 1,
  'you': 1},
 'wow': 0}

diaper_champ_reviews['selected_predicted_sentiment'] = selected_words_model.predict(diaper_champ_reviews,output_type='probability')

diaper_champ_reviews.head()