Trouble reading CSV into Django models with python DictReader

Question

I'm using Cloud9 (which from what I've read uses Python 2.6 currently, not Python 3) to write a Django app. I'm trying to read a CSV file with DictReader and use each column in the CSV to create a new instance of a model and populate the model fields.

views.py

class GenerateFromCSV(CreateView):
    model = group
    template_name = "my_app/csv_generate.html"
    def form_valid(self, form):
        new_group = form.save()
        the_csv = open(new_group.group_csv, 'rbU')
        fieldnames = ['c_type', 'f_name', 'q_type', 'ans', 'n_questions', 'bucket']
        data_file = csv.DictReader(the_csv, fieldnames = fieldnames, delimiter=',', dialect=csv.excel)
        for row in data_file:
            new_card = Card(
                name = 'card',
                card_type = row['c_type'], 
                file_name = row['f_name'], 
                question_type = row['q_type'], 
                answer = row['ans'], 
                num_questions = row['n_questions'], 
                bucket = row['bucket'],
                exam = new_exam)
            new_card.save()

models.py

class Group(models.Model):
    name = models.CharField(max_length=255, blank = True)
    subject = models.CharField(max_length=255, choices = SUBJECT, blank = True)
    num_questions = models.IntegerField(default=0, blank = True, null = True)
    group_csv = models.FileField(upload_to='csv', blank = True, null = True)
    def __unicode__(self):
        return self.name

class Card(models.Model):
    name = models.CharField(max_length=255, blank = True)
    #ordered same as column order in CSV
    card_type = models.CharField(max_length=255, choices = CARDTYPE, blank = True)
    file_name = models.CharField(max_length=255, blank = True)
    question_type = models.IntegerField(default = 0, blank = True, null = True)
    answer = models.IntegerField(max_length = 1, choices = ANSWERS, blank = True, null = True)
    num_questions = models.IntegerField(default = 0, blank = True, null = True)
    bucket = models.CharField(max_length=255, blank = True)
    exam = models.ForeignKey(Exam)
    def __unicode__(self):
        return self.name or 'card'

With the code as it is above, I get a TypeError (coercing to Unicode: need string or buffer, FieldFile found) when I call open() on the CSV. If I remove the call to open(), I get the error: 'new-line character seen in unquoted field - do you need to open the file in universal-newline mode?'

My CSV is in the format (not every column contains data in every row):

3,the_file_name.png,0,"00001",,Equations

What is the correct syntax for this?

Edit Here's my stacktrace:

Traceback:
File "/usr/libexec/openshift/cartridges/c9-0.1/root/python2.6.6/site-packages/Django-1.5-py2.6.egg/django/core/handlers/base.py" in get_response
  115.                         response = callback(request, *callback_args, **callback_kwargs)
File "/usr/libexec/openshift/cartridges/c9-0.1/root/python2.6.6/site-packages/Django-1.5-py2.6.egg/django/views/generic/base.py" in view
  68.             return self.dispatch(request, *args, **kwargs)
File "/usr/libexec/openshift/cartridges/c9-0.1/root/python2.6.6/site-packages/Django-1.5-py2.6.egg/django/views/generic/base.py" in dispatch
  86.         return handler(request, *args, **kwargs)
File "/usr/libexec/openshift/cartridges/c9-0.1/root/python2.6.6/site-packages/Django-1.5-py2.6.egg/django/views/generic/edit.py" in post
  199.         return super(BaseCreateView, self).post(request, *args, **kwargs)
File "/usr/libexec/openshift/cartridges/c9-0.1/root/python2.6.6/site-packages/Django-1.5-py2.6.egg/django/views/generic/edit.py" in post
  165.             return self.form_valid(form)
File "/var/lib/stickshift/52a55ef4e0b8cde0ff000036/app-root/data/705411/zamrdjango/zamr/views.py" in form_valid
  35.         with new_exam.exam_csv.open('rbU') as the_csv:

Exception Type: AttributeError at /import/
Exception Value: 'NoneType' object has no attribute '__exit__'

what's new_item = Item( did you mean new_card = Card instead ? — Guy Gavriely
– Guy Gavriely, Commented Jan 13, 2014 at 17:29

ekrah · Accepted Answer · 2014-01-15 22:17:30Z

Well, I ended up fixing it by instead calling .read() on the csv and then calling .splitlines() to get the rows and columns for each row. Here's my new views.py:

def form_valid(self, form):
    new_group = form.save()
    print(new_exam.exam_csv.name)
    data = new_group.group_csv.read()
    rows = data.splitlines()
    for row in rows:
        columns = row.split(',')
        print(columns)
        new_card = Card(
            name = columns[0],
            card_type = columns[0], 
            file_name = columns[1], 
            question_type = int(columns[2]),
            answer = columns[3].replace('"', '').strip(), 
            num_questions = columns[4], 
            bucket = columns[5],
            group = new_group)
        print(new_card.name)
        new_card.save()

I'm certain that this isn't the correct syntax or convention, and there are still a few things to figure out (like column names) but for now it's correctly parsing all the information into the correct model fields.

Anentropic · Accepted Answer · 2014-01-13 17:29:22Z

0

Have a look at the docs for FileField since the problem is you are not passing a filename to the file open function:
https://docs.djangoproject.com/en/dev/ref/models/fields/#django.db.models.fields.files.FieldFile

Your code could instead be:

class GenerateFromCSV(CreateView):
    model = group
    template_name = "my_app/csv_generate.html"
    def form_valid(self, form):
        new_group = form.save()
        with new_group.group_csv.open('rbU') as the_csv:
            fieldnames = ['c_type', 'f_name', 'q_type', 'ans', 'n_questions', 'bucket']
            data_file = csv.DictReader(the_csv, fieldnames = fieldnames, delimiter=',', dialect=csv.excel)
            for row in data_file:
                new_card = Card(
                    name = 'item',
                    card_type = row['c_type'], 
                    file_name = row['f_name'], 
                    question_type = row['q_type'], 
                    answer = row['ans'], 
                    num_questions = row['n_questions'], 
                    bucket = row['bucket'],
                    exam = new_exam)
                new_card.save()

answered Jan 13, 2014 at 17:29

Anentropic

34.2k13 gold badges111 silver badges170 bronze badges

4 Comments

ekrah Over a year ago

I see, I think I was misunderstanding Python with/as statement. I'm now getting the attribute error: 'NoneType' object has no attribute 'exit'. I'm unsure how to handle this, is the context handler trying to call exit on the csv file but is unable to?

Anentropic Over a year ago

yes __exit__ is called on the_csv file object at the end of the with block... it is odd that you should get that error if the_csv is None though, seems like another exception would be raised earlier. can you maybe post a full stack trace?

ekrah Over a year ago

Okay, I added the traceback above. I also tried making sure that every column in every row had data, in case that was the issue, and trying a different csv (in case the one I was using was invalid for whatever reason) but I'm still getting the same attribute error.

Anentropic Over a year ago

ok, it's failing early, so it looks like the problem is simply that your new_group.group_csv field has a None value instead of a file

Harry Moreno · Accepted Answer · 2021-05-05 22:46:31Z

I did something like this

{# csv-import.html %}
<form action="/csv-import" method="POST" enctype="multipart/form-data">
    {% csrf_token %}
    <input type="file" required name="csvfile" id="id_csvfile" />
    <input type="submit" />
</form>

# views.py
@login_required
def CSVImport(request):
    if request.method == "GET":
        logger.info(f"Get /csv-import")
    elif request.method == "POST":
        logger.info(f"POST /csv-import")
        csv_len = 0
        try:
            csv_file = request.FILES["csvfile"]
            if not csv_file.name.endswith('.csv'):
                messages.error(request,'File is not CSV type')
                return HttpResponseRedirect(reverse("myapp:upload_csv"))
            #if file is too large, return
            if csv_file.multiple_chunks():
                messages.error(request, "Uploaded file is too big (%.2f MB)." % (csv_file.size/(1000*1000),))
                return HttpResponseRedirect(reverse("myapp:upload_csv"))
            file_data = csv.DictReader(csv_file.read().decode('utf-8').splitlines())
            for row in file_data:
                print('row', row)
                csv_len += 1

        except Exception as e:
            logging.getLogger("error_logger").error("Unable to upload file. "+repr(e))
            messages.error(request,"Unable to upload file. "+repr(e))
        messages.add_message(request, messages.INFO, f"Updated {csv_len} meetings.")
    return render(request, "csv-import.html")

Collectives™ on Stack Overflow

Trouble reading CSV into Django models with python DictReader

3 Answers 3

Comments

4 Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

3 Answers 3

Comments

4 Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related