Analyze functionality now in new class Agent

ADA/DataLoader.py

@@ -12,4 +12,4 @@ class DataLoader:
         return self.get_reviews(product_id)['product_title'][0]
 
     def get_avg_star_rating(self, product_id):
-        return self.get_reviews(product_id)['star_rating'].mean()
+        return float(self.get_reviews(product_id)['star_rating'].mean())

ADA/agent.py

+from nltk.tokenize import sent_tokenize
+import re
+from review_tokenizer import ReviewTokenizer
+from anytree import PostOrderIter
+import pickle
+from argument import *
+from functools import reduce
+
+class Agent:
+
+    sentiment_threshold = 0.95
+    review_tokenizer = ReviewTokenizer()
+
+    def __init__(self):
+        # load classifier
+        f = open('camera_review_classifier.pickle', 'rb')
+        self.classifier = pickle.load(f)
+        f.close()
+
+    # extract phrases
+    def extract_phrases(self, review_body):
+        sentences = sent_tokenize(review_body)
+        phrases = []
+        for sentence in sentences:
+            phrases += re.split(' but | although | though | otherwise | however | unless | whereas | despite |<br />', sentence)
+        return phrases
+
+    # analyze sentiment
+    def get_sentiment(self, phrase):
+        # get classification
+        tokens = self.review_tokenizer.tokenize_review(phrase)
+        prob_classification = self.classifier.prob_classify(dict([token, True] for token in tokens))
+        classification = prob_classification.max()
+        strength = (prob_classification.prob(classification) - 0.5) * 2
+        return strength if classification == '+' else -strength
+
+    # remove all ancestors of node in list l
+    def remove_ancestors(self, node, l):
+        if node.parent != None:
+            try:
+                l.remove(node.parent)
+            except ValueError:
+                pass
+            self.remove_ancestors(node.parent, l)
+
+    # get argument(s) that match phrase
+    def get_arguments(self, phrase):
+        argument_matches = []
+        arguments = [node for node in PostOrderIter(camera)]
+        while len(arguments) > 0:
+            f = arguments.pop(0)
+            for word in glossary[f]:
+                if word in phrase:
+                    argument_matches.append(f)
+                    self.remove_ancestors(f, arguments)
+                    break
+        return argument_matches
+
+    def extract_votes(self, phrases):
+        votes = {}
+        for phrase in phrases:
+            arguments = self.get_arguments(phrase)
+            sentiment = self.get_sentiment(phrase)
+            if abs(sentiment) > self.sentiment_threshold:
+                for argument in arguments:
+                    if (argument not in votes) or (abs(votes[argument]) < abs(sentiment)):
+                        votes[argument] = sentiment # what if there's two phrases with same argument?
+        # normalize votes to 1 (+) or -1 (-)
+        for argument in votes:
+            votes[argument] = 1 if votes[argument] > 0 else -1
+        return votes
+
+    # augment votes (Definition 4.3) obtained for a single critic
+    def augment_votes(self, votes):
+        arguments = [node for node in PostOrderIter(camera)]
+        for argument in arguments:
+            if argument not in votes:
+                polar_sum = 0
+                for subfeat in argument.children:
+                    if subfeat in votes:
+                        polar_sum += votes[subfeat]
+                if polar_sum != 0:
+                    votes[argument] = 1 if polar_sum > 0 else -1
+
+    def get_qbaf(self, ra, review_count):
+        # sums of all positive and negative votes for arguments
+        argument_sums = {}
+        for argument in arguments:
+            argument_sums[argument] = 0
+            for r in ra:
+                if r['argument'] == argument:
+                    argument_sums[argument] += r['vote']
+
+        # calculate attack/support relations for camera
+        supporters = {r: [] for r in arguments}
+        attackers = {r: [] for r in arguments}
+        for r in arguments:
+            for subf in r.children:
+                if argument_sums[subf] > 0:
+                    supporters[r].append(subf)
+                elif argument_sums[subf] < 0:
+                    attackers[r].append(subf)
+
+        # calculate base scores for arguments
+        base_scores = {}
+        base_scores[camera] = 0.5 + 0.5 * argument_sums[camera] / review_count
+        for feature in features:
+            base_scores[feature] = abs(argument_sums[feature]) / review_count
+
+        qbaf = {"supporters": supporters, "attackers": attackers, "base_scores": base_scores}
+        return qbaf
+
+    def combined_strength(self, args):
+        if len(args) != 0:
+            return 1 - reduce(lambda x, y: x * y, map(lambda v: 1 - v, args))
+        return 0
+
+    def argument_strength(self, base_score, attacker_strengths, supporter_strengths):
+        attack = self.combined_strength(attacker_strengths)
+        support = self.combined_strength(supporter_strengths)
+        if attack > support:
+            return base_score - (base_score * abs(attack - support))
+        elif attack < support:
+            return base_score + ((1 - base_score) * abs(attack - support))
+        return base_score
+
+    # apply DF-QUAD gradual semantics to qbaf
+    def get_strengths(self, qbaf):
+        strengths = {}
+        arguments = [node for node in PostOrderIter(camera)]
+        for argument in arguments:
+            attacker_strengths = []
+            supporter_strengths = []
+            for child in argument.children:
+                if child in qbaf["attackers"][argument]:
+                    attacker_strengths.append(strengths[child])
+                elif child in qbaf["supporters"][argument]:
+                    supporter_strengths.append(strengths[child])
+            strengths[argument] = self.argument_strength(qbaf["base_scores"][argument], attacker_strengths, supporter_strengths)
+        return strengths
+
+    def analyze_reviews(self, reviews):
+        # get ra
+        self.ra = []
+        voting_reviews = 0
+        review_count = 0
+        for _, review in reviews.iterrows():
+            review_id = review['review_id']
+            review_count += 1
+            phrases = self.extract_phrases(review['review_body'])
+            votes = self.extract_votes(phrases)
+            self.augment_votes(votes)
+            voting_reviews += 1 if len(votes) > 0 else 0
+            # add final vote tuples to ra with simplified polarity in {+ (true), - (false)}
+            for argument in votes:
+                self.ra.append({'review_id': review_id, 'argument': argument, 'vote': votes[argument]})
+        # only consider items that obtained votes from at least 33% of reviewers
+        if voting_reviews / review_count < 0.33:
+            print('warning: only a small fraction of reviews generated votes')
+        # get qbaf from ra
+        self.qbaf = self.get_qbaf(self.ra, review_count)
+        # apply gradual semantics
+        self.strengths = self.get_strengths(self.qbaf)
+        # print results
+        print('qbaf:')
+        print(self.qbaf)
+        print('strengths:')
+        print(self.strengths)

ADA/analyze_data.py

@@ -10,7 +10,21 @@ from scipy.stats import pearsonr
 from sklearn.metrics import mean_absolute_error
 import pickle
 from review_tokenizer import tokenize_review, reduce_noise
-from item import *
+from argument import *
+
+reviewables = [camera, image, video, battery, flash, audio, price, shipping, lens, zoom, af]
+features = [image, video, battery, flash, audio, price, shipping, lens, zoom, af]
+
+glossary = {
+    camera: ['camera', 'device', 'product'],
+    image: ['image', 'picture', ' pic '],
+    video: ['video'],
+    battery: ['battery'],
+    flash: ['flash'],
+    audio: ['audio', 'sound'],
+    price: ['price', 'value', 'cost'],
+    shipping: ['ship']
+}
 
 sentiment_threshold = 0.95
 

ADA/argument.py

+from anytree import Node
+
+camera = Node('camera')
+image = Node('image', parent=camera)
+video = Node('video', parent=camera)
+battery = Node('battery', parent=camera)
+flash = Node('flash', parent=camera)
+audio = Node('audio', parent=camera)
+price = Node('price', parent=camera)
+shipping = Node('shipping', parent=camera)
+lens = Node('lens', parent=camera)
+zoom = Node('zoom', parent=lens)
+af = Node('af', parent=lens)
+
+arguments = [camera, image, video, battery, flash, audio, price, shipping, lens, zoom, af]
+features = [image, video, battery, flash, audio, price, shipping, lens, zoom, af]
+
+glossary = {
+    camera: ['camera', 'device', 'product'],
+    image: ['image', 'picture', ' pic '],
+    video: ['video'],
+    battery: ['battery'],
+    flash: ['flash'],
+    audio: ['audio', 'sound'],
+    price: ['price', 'value', 'cost'],
+    shipping: ['ship'],
+    lens: ['lens'],
+    zoom: ['zoom'],
+    af: ['autofocus', 'auto-focus']
+}
+
+class Argument:
+
+    def __init__(self, id, name):
+        self.id = id
+        self.name = name
+        self.queries = []
+
+    def withQueries(self, queries):
+        arg = Argument(self.id, self.name)
+        arg.queries = queries
+        return arg

ADA/argumentquery.py

+class ArgumentQuery:
+
+    def __init__(self, queryID, text):
+        self.queryID = queryID
+        self.text = text
+
+    def withArgument(self, argument):
+        query = ArgumentQuery(self.queryID, self.text)
+        query.argumentID = argument.id
+        query.text = query.text.format(arg=argument.name)
+        return query

ADA/communicator.py

+from argumentquery import ArgumentQuery
+from dataloader import DataLoader
+from argument import *
+from agent import Agent
+
+class ADAMessage:
+
+    def __init__(self, text, arguments):
+        self.text = text
+        self.arguments = arguments
+
+class Communicator:
+
+    queries = [
+        ArgumentQuery(1, 'Why was the {arg} highly rated?'),
+        ArgumentQuery(2, 'Why was the {arg} poorly rated?')
+    ]
+
+    product = camera # node
+    agent = Agent()
+
+    def __init__(self, product_id, dl):
+        self.product_id = product_id
+        self.dl = dl
+        self.arguments = {arguments[i] : Argument(i, arguments[i].name) for i in range(len(arguments))}
+        self.agent.analyze_reviews(self.dl.get_reviews(self.product_id))
+
+    def get_init_message(self):
+        text = 'What would you like to know about the {}?'.format(self.arguments[self.product].name)
+        if self.dl.get_avg_star_rating(self.product_id) > 3:
+            queries = [self.queries[0].withArgument(self.arguments[self.product])]
+        else:
+            queries = [self.queries[1].withArgument(self.arguments[self.product])]
+        args = [self.arguments[self.product].withQueries(queries)]
+        return ADAMessage(text, args)
+
+    def get_response(self, arg_id, query_id):
+        return 0

ADA/review_tokenizer.py

@@ -3,13 +3,15 @@ from nltk.corpus import stopwords
 from item import glossary
 import string
 
-tokenizer = TweetTokenizer()
-stop_words = stopwords.words('english')
-flat_glossary = [val for sublist in list(glossary.values()) for val in sublist]
+class ReviewTokenizer:
 
-def tokenize_review(review):
-    return tokenizer.tokenize(review)
+    tokenizer = TweetTokenizer()
+    stop_words = stopwords.words('english')
+    flat_glossary = [val for sublist in list(glossary.values()) for val in sublist]
 
-def reduce_noise(tokens):
-    lowercase_tokens = list(map(lambda s: s.lower(), tokens))
-    return list(filter(lambda s: len(s) > 0 and s not in string.punctuation and s not in stop_words and s not in flat_glossary, lowercase_tokens))
+    def tokenize_review(self, review):
+        return self.reduce_noise(self.tokenizer.tokenize(review))
+
+    def reduce_noise(self, tokens):
+        lowercase_tokens = list(map(lambda s: s.lower(), tokens))
+        return list(filter(lambda s: len(s) > 0 and s not in string.punctuation and s not in self.stop_words and s not in self.flat_glossary, lowercase_tokens))

ADA/server/ios_server/views.py

 from django.http import JsonResponse, HttpResponse
 import json
+import jsonpickle
 from django.views.decorators.csrf import csrf_exempt
 
 import sys
 sys.path.append('/Users/joeloksanen/individual_project/ADA')
-from DataLoader import DataLoader
+from dataloader import DataLoader
+from communicator import Communicator
 
 dl = DataLoader()
 
@@ -13,10 +15,27 @@ def index(request):
 
 def product(request):
     id = request.GET.get('id', '')
+
     name = dl.get_product_name(id)
     star_rating = dl.get_avg_star_rating(id)
-    imageURL = 'https://ws-na.amazon-adsystem.com/widgets/q?_encoding=UTF8&MarketPlace=US&ASIN=' + id + '&ServiceVersion=20070822&ID=AsinImage&WS=1&Format=SL250'
-    return JsonResponse({'id': id, 'name': name, 'starRating': star_rating, 'imageURL': imageURL})
+    image_url = 'https://ws-na.amazon-adsystem.com/widgets/q?_encoding=UTF8&MarketPlace=US&ASIN=' + id + '&ServiceVersion=20070822&ID=AsinImage&WS=1&Format=SL250'
+
+    communicator = Communicator(id, dl)
+    init_message = communicator.get_init_message()
+
+    class Empty:
+        pass
+
+    product_info = Empty()
+    product_info.id = id
+    product_info.name = name
+    product_info.starRating = star_rating
+    product_info.imageURL = image_url
+    init_response = Empty()
+    init_response.productInfo = product_info
+    init_response.message = init_message
+
+    return HttpResponse(jsonpickle.encode(init_response, unpicklable=False), content_type="application/json")
 
 @csrf_exempt
 def message(request):

ADAbot/ADAbot.xcodeproj/project.pbxproj

@@ -30,6 +30,7 @@
 		9449FE4F240533FD00025F70 /* QueryOptionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9449FE4E240533FD00025F70 /* QueryOptionView.swift */; };
 		9449FE5124053DA500025F70 /* ChatManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9449FE5024053DA500025F70 /* ChatManager.swift */; };
 		94BE1EEB2407E26900741749 /* RatingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94BE1EEA2407E26900741749 /* RatingView.swift */; };
+		94BE1EED240800D800741749 /* InitResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94BE1EEC240800D800741749 /* InitResponse.swift */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXFileReference section */
@@ -58,6 +59,7 @@
 		9449FE4E240533FD00025F70 /* QueryOptionView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QueryOptionView.swift; sourceTree = "<group>"; };
 		9449FE5024053DA500025F70 /* ChatManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatManager.swift; sourceTree = "<group>"; };
 		94BE1EEA2407E26900741749 /* RatingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RatingView.swift; sourceTree = "<group>"; };
+		94BE1EEC240800D800741749 /* InitResponse.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InitResponse.swift; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -94,6 +96,7 @@
 				9449FE1C2402C84F00025F70 /* SceneDelegate.swift */,
 				9449FE1E2402C84F00025F70 /* ContentView.swift */,
 				9449FE342402CCDA00025F70 /* ConnectionManager.swift */,
+				94BE1EEC240800D800741749 /* InitResponse.swift */,
 				9449FE3F2403EDD300025F70 /* Product */,
 				9449FE3E2403EDB400025F70 /* Chat */,
 				9449FE202402C85300025F70 /* Assets.xcassets */,
@@ -232,6 +235,7 @@
 				9449FE352402CCDA00025F70 /* ConnectionManager.swift in Sources */,
 				9449FE1D2402C84F00025F70 /* SceneDelegate.swift in Sources */,
 				9449FE1F2402C84F00025F70 /* ContentView.swift in Sources */,
+				94BE1EED240800D800741749 /* InitResponse.swift in Sources */,
 				94BE1EEB2407E26900741749 /* RatingView.swift in Sources */,
 				9449FE332402CBD400025F70 /* ProductInfo.swift in Sources */,
 			);

ADAbot/ADAbot/Chat/Argument.swift

@@ -12,8 +12,7 @@ struct Argument: Codable, Equatable {
   
   let id = UUID()
   let name: String
-  let queries = [ArgumentQuery(argumentID: 1, queryID: 1, text: "Why was the lens considered to be good?"),
-                 ArgumentQuery(argumentID: 1, queryID: 2, text: "What did users say about the lens being good?")]
+  let queries: [ArgumentQuery]
   
   static func == (lhs: Argument, rhs: Argument) -> Bool {
     lhs.id == rhs.id

ADAbot/ADAbot/Chat/ChatManager.swift

@@ -23,7 +23,7 @@ class ChatManager: ObservableObject {
     sent = true
     DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) {
       self.sent = false
-      self.connectionManager.addMessage(message: query.message)
+      self.connectionManager.addMessage(query.message)
       self.clearOptions()
     }
   }

ADAbot/ADAbot/Chat/ChatView.swift

@@ -22,13 +22,15 @@ struct ChatView: View {
       Rectangle()
         .foregroundColor(Color(red: 231/255, green: 234/255, blue: 239/255))
       
-      ScrollView {
-        VStack(spacing: 0) {
-          ForEach(connectionManager.messages, id: \.id) { message in
-            MessageView(chatManager: self.chatManager, message: message)
+      if !connectionManager.messages.isEmpty {
+        ScrollView {
+          VStack(spacing: 0) {
+            ForEach(connectionManager.messages, id: \.id) { message in
+              MessageView(chatManager: self.chatManager, message: message)
+            }
           }
+          .padding(EdgeInsets(top: 30, leading: 0, bottom: 30, trailing: 0))
         }
-        .padding(EdgeInsets(top: 30, leading: 0, bottom: 30, trailing: 0))
       }
     }
   }

ADAbot/ADAbot/ConnectionManager.swift

@@ -10,8 +10,9 @@ import SwiftUI
 
 class ConnectionManager: ObservableObject {
   @Published var product = Product()
-  @Published var messages: [Message] = [UserMessage(text: "Why was the camera highly rated?"),
-                                        ADAMessage(text: "The camera was highly rated because the lens was good, although the battery was poor.", arguments: [Argument(name: "camera"), Argument(name: "lens"), Argument(name: "battery")])]
+  @Published var messages: [Message] = [Message]()
+//  [UserMessage(text: "Why was the camera highly rated?"),
+//  ADAMessage(text: "The camera was highly rated because the lens was good, although the battery was poor.", arguments: [Argument(name: "camera"), Argument(name: "lens"), Argument(name: "battery")])
   
   init() {
     requestProduct(id: "B00RTGK0N0")
@@ -23,12 +24,12 @@ class ConnectionManager: ObservableObject {
     let task = URLSession.shared.dataTask(with: url) {(data, response, error) in
       guard let data = data else { return }
       do {
-        let productInfo = try JSONDecoder().decode(ProductInfo.self, from: data)
+        let resp = try JSONDecoder().decode(InitResponse.self, from: data)
         DispatchQueue.main.async {
-          self.requestImage(at: productInfo.imageURL)
-          self.product.name = productInfo.name
-          self.product.starRating = productInfo.starRating
-          print(productInfo.starRating)
+          self.requestImage(at: resp.productInfo.imageURL)
+          self.product.name = resp.productInfo.name
+          self.product.starRating = resp.productInfo.starRating
+          self.messages.append(resp.message)
         }
       } catch let parseError {
         print(parseError)
@@ -85,7 +86,7 @@ class ConnectionManager: ObservableObject {
     task.resume()
   }
   
-  func addMessage(message: Message) {
+  func addMessage(_ message: Message) {
     messages.append(message)
   }
   

ADAbot/ADAbot/InitResponse.swift

+//
+//  InitResponse.swift
+//  ADAbot
+//
+//  Created by Joel Oksanen on 27.2.2020.
+//  Copyright © 2020 Joel Oksanen. All rights reserved.
+//
+
+struct InitResponse: Decodable {
+  
+  let productInfo: ProductInfo
+  let message: ADAMessage
+  
+}

ADAbot/ADAbot/Product/RatingView.swift

@@ -30,7 +30,7 @@ struct RatingView: View {
   }
   
   var body: some View {
-    HStack(spacing: 3) {
+    HStack(spacing: 1) {
       ForEach(0..<fullStars, id: \.self) { i in
         ZStack {
           Image("star_fill")