From b7243dcce0691ad33ef77427744f2436aa235643 Mon Sep 17 00:00:00 2001
From: Ben Adida <ben@adida.net>
Date: Mon, 22 Feb 2021 02:05:08 +0000
Subject: [PATCH] allow for uploading of a latin1-encoded voter file, not just
 utf-8

---
 helios/fixtures/voter-file-latin1.csv |  4 ++++
 helios/models.py                      | 12 ++++++++----
 helios/tests.py                       |  7 +++++++
 3 files changed, 19 insertions(+), 4 deletions(-)
 create mode 100644 helios/fixtures/voter-file-latin1.csv

diff --git a/helios/fixtures/voter-file-latin1.csv b/helios/fixtures/voter-file-latin1.csv
new file mode 100644
index 0000000..fdbc9b3
--- /dev/null
+++ b/helios/fixtures/voter-file-latin1.csv
@@ -0,0 +1,4 @@
+ benadida5,ben5@adida.net ,  Ben5 Adida 
+benadida6,ben6@adida.net,Ben6 Adida
+benadida7,ben7@adida.net,Ben7 Adida
+testlatin1,Test Latin1,J�NIO LUIZ CORREIA J�NIOR
diff --git a/helios/models.py b/helios/models.py
index 8334407..7ccadea 100644
--- a/helios/models.py
+++ b/helios/models.py
@@ -272,11 +272,15 @@ class Election(HeliosModel):
     """
     expects a django uploaded_file data structure, which has filename, content, size...
     """
-    # now we're just storing the content
-    # random_filename = str(uuid.uuid4())
-    # new_voter_file.voter_file.save(random_filename, uploaded_file)
+    voter_file_content_bytes = uploaded_file.read()
 
-    new_voter_file = VoterFile(election = self, voter_file_content = uploaded_file.read())
+    # usually it's utf-8 encoded, but occasionally it's latin-1
+    try:
+      voter_file_content = voter_file_content_bytes.decode('utf-8')
+    except:
+      voter_file_content = voter_file_content_bytes.decode('latin-1')
+    
+    new_voter_file = VoterFile(election = self, voter_file_content = voter_file_content)
     new_voter_file.save()
     
     self.append_log(ElectionLog.VOTER_FILE_ADDED)
diff --git a/helios/tests.py b/helios/tests.py
index 60103c8..97d7dba 100644
--- a/helios/tests.py
+++ b/helios/tests.py
@@ -585,6 +585,13 @@ class ElectionBlackboxTests(WebTest):
         response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'confirm_p': "1"})
         self.assertRedirects(response, "/helios/elections/%s/voters/list" % election_id)
 
+        # Try a latin-1 encoded file
+        FILE = "helios/fixtures/voter-file-latin1.csv"
+        voters_file = open(FILE, mode='rb')
+        response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'voters_file': voters_file})
+        voters_file.close()
+        self.assertContains(response, "first few rows of this file")
+        
         # and we want to check that there are now voters
         response = self.client.get("/helios/elections/%s/voters/" % election_id)
         NUM_VOTERS = 4
-- 
GitLab