improved unicode handling

helios/fixtures/voter-file.csv

  benadida5,ben5@adida.net ,  Ben5 Adida 
 benadida6,ben6@adida.net,Ben6 Adida
 benadida7,ben7@adida.net,Ben7 Adida
-benadida8,ben8@adida.net,Ben8 Adida
+ernesto,helios-testing-ernesto@adida.net,Erñesto Testing Helios
\ No newline at end of file

helios/models.py

@@ -11,7 +11,7 @@ from django.utils import simplejson
 from django.conf import settings
 from django.core.mail import send_mail
 
-import datetime, logging, uuid, random, StringIO, io
+import datetime, logging, uuid, random, io
 
 from crypto import electionalgs, algs, utils
 from helios import utils as heliosutils
@@ -668,7 +668,12 @@ class VoterFile(models.Model):
 
   def itervoters(self):
     if self.voter_file_content:
-      voter_stream = io.StringIO(unicode(self.voter_file_content), newline=None)
+      if type(self.voter_file_content) == unicode:
+        content = self.voter_file_content.encode('utf-8')
+      else:
+        content = self.voter_file_content
+
+      voter_stream = io.BytesIO(content, newline=None)
     else:
       voter_stream = open(self.voter_file.path, "rU")
 
@@ -680,13 +685,13 @@ class VoterFile(models.Model):
       if len(voter_fields) < 1:
         continue
     
-      return_dict = {'voter_id': voter_fields[0]}
+      return_dict = {'voter_id': voter_fields[0].strip()}
 
       if len(voter_fields) > 1:
-        return_dict['email'] = voter_fields[1]
+        return_dict['email'] = voter_fields[1].strip()
 
       if len(voter_fields) > 2:
-        return_dict['name'] = voter_fields[2]
+        return_dict['name'] = voter_fields[2].strip()
 
       yield return_dict
     
@@ -695,50 +700,24 @@ class VoterFile(models.Model):
     self.save()
 
     election = self.election    
-
-    # now we're looking straight at the content
-    if self.voter_file_content:
-      voter_stream = io.StringIO(unicode(self.voter_file_content), newline=None)
-    else:
-      voter_stream = open(self.voter_file.path, "rU")
-
-    # reader = unicode_csv_reader(voter_stream)
-    reader = unicodecsv.reader(voter_stream, encoding='utf-8')
-    
     last_alias_num = election.last_alias_num
 
     num_voters = 0
     new_voters = []
-    for voter in reader:
-      # bad line
-      if len(voter) < 1:
-        continue
-    
+    for voter in self.itervoters():
       num_voters += 1
-      voter_id = voter[0].strip()
-      name = voter_id
-      email = voter_id
-    
-      if len(voter) > 1:
-        email = voter[1].strip()
-    
-      if len(voter) > 2:
-        name = voter[2].strip()
-    
-      # create the user -- NO MORE
-      # user = User.update_or_create(user_type='password', user_id=email, info = {'name': name})
     
       # does voter for this user already exist
-      voter = Voter.get_by_election_and_voter_id(election, voter_id)
+      existing_voter = Voter.get_by_election_and_voter_id(election, voter['voter_id'])
     
       # create the voter
-      if not voter:
+      if not existing_voter:
         voter_uuid = str(uuid.uuid4())
-        voter = Voter(uuid= voter_uuid, user = None, voter_login_id = voter_id,
-                      voter_name = name, voter_email = email, election = election)
-        voter.generate_password()
-        new_voters.append(voter)
-        voter.save()
+        existing_voter = Voter(uuid= voter_uuid, user = None, voter_login_id = voter['voter_id'],
+                      voter_name = voter['name'], voter_email = voter['email'], election = election)
+        existing_voter.generate_password()
+        new_voters.append(existing_voter)
+        existing_voter.save()
 
     if election.use_voter_aliases:
       voter_alias_integers = range(last_alias_num+1, last_alias_num+1+num_voters)

helios/tests.py

@@ -544,6 +544,9 @@ class ElectionBlackboxTests(WebTest):
         self.assertContains(response, "Trustee #1")
 
         # add a few voters, via file upload
+        # this file now includes a UTF-8 encoded unicode character
+        # yes I know that's not how you spell Ernesto.
+        # I just needed some unicode quickly.
         FILE = "helios/fixtures/voter-file.csv"
         voters_file = open(FILE)
         response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'voters_file': voters_file})