import re
import os
import csv
import urllib2
import certifi
import urllib3
import json
import simplejson
import socket
from collections import namedtuple
from datetime import datetime
from math import pow
from copy import deepcopy
from django.http import Http404, HttpResponse, \
HttpResponseRedirect, JsonResponse, HttpResponseNotFound, \
FileResponse
from django.core.files import File
from django.core import serializers
from django.core.serializers import serialize
from django_tables2 import RequestConfig
from django.contrib.auth.models import User
from django.contrib.sites.shortcuts import get_current_site
from django.core.files.temp import NamedTemporaryFile
from django.shortcuts import get_object_or_404, render
from django.contrib.auth import authenticate
from django.contrib.auth import login as auth_login
from django.contrib.auth.decorators import login_required, user_passes_test
from django.views.decorators.csrf import ensure_csrf_cookie
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.db.models import ForeignKey, FloatField, \
IntegerField, BooleanField, Q, DateField, Count
from django.core.mail import send_mail
from django.conf import settings
from django.core.cache import cache
from django.views.decorators.cache import cache_page
from urllib3.exceptions import InsecureRequestWarning
import StringIO
import tarfile
# from django.core.servers.basehttp import FileWrapper
from wsgiref.util import FileWrapper
from .tables import SeriesTable, BioTable, AssayTable, \
AppliedAssayTable, SequencingTable, DataTable, BatchUploadTable,\
StatusesTable, BiosampleTypesTable, AnatomicalTermsTable, \
LabsTable, GeneticBackgroundsTable, DevelopmentalStagesTable, \
SexesTable, AssayTypesTable, PlatformsTable, InstrumentsTable, \
StrandModesTable
from .models import Series, Biosample, BiosampleReplicate, Assay, \
AppliedAssay, TechnicalReplicate, Sequencing, Data, BatchUploads, \
BatchUploadDetails, statuses, biosample_types, labs, \
anatomical_terms, genetic_backgrounds, developmental_stages, \
sexes, assay_types, platforms, instruments, strand_modes, \
file_types, mapped_genomes, Versions, generate_sha
from .forms import UserForm, BatchUploadForm
UPLOAD_PATH = settings.UPLOAD_PATH
urllib3.disable_warnings(InsecureRequestWarning)
BATCH_FALSE_VALUES = set((
'False', 'FALSE', 'false', 'F', 'f',
'No', 'no', 'N', 'n'))
INITIAL_UPLOAD_STATUS = "initial entry"
COMPLETED_UPLOAD_STATUS = "complete upload"
WITH_ASSOCIATED_FILES = "with processed files"
VALID_BATCH_HEADERS = [
f.name for f in BatchUploadDetails._meta.get_fields()
if f.name not in (
'id', 'batchUpload', 'row_type', 'errors')]
RNA_BASED_ASSAYS = (
'RNA-seq', 'short-RNA-seq', 'miRNA-seq', 'CAGE-seq', 'Bru-seq', 'PAS-seq',
'Ribo-seq', 'RIP-seq', 'PAR-Clip-seq', 'iCLIP-seq', 'GRO-seq', 'PAL-seq',
'3P-seq', 'SAPAS-seq')
TARGET_REQD_ASSAYS = (
'ChIP-seq', 'SELEX-seq', 'RIP-seq', 'PAR-Clip-seq',
'iCLIP-seq', 'ChIP-exo-seq', 'Methyl-seq')
ID_FIELD = namedtuple('Field_id', ['name', 'value'])
GROUP = settings.GROUP
[docs]def is_member(user):
"""
Check if `user` is part of GROUP.
Args:
`user` (User)
Returns:
``True`` if `user` is part of GROUP
"""
return user.groups.filter(name=GROUP).exists()
[docs]def index(request):
"""
Render the index page.
"""
return render(request, 'modeldcc/index.html')
[docs]def policyPage(request):
"""
Render the data policy page.
"""
return render(request, 'modeldcc/policy.html')
[docs]def announcements(request):
"""
Render the announcements page.
"""
return render(request, 'modeldcc/announcements.html')
# def login(request):
# return render(request, 'registration/login.html')
[docs]def login_interface(request):
"""
Log in as a user with a POST request (mainly used for cypress tests).
Returns:
"fine", if login is successful
"incactive", if user is not activated yet
"bad", if login was unsuccessful
"""
username = request.POST.get('username')
password = request.POST.get('password')
user = authenticate(username=username, password=password)
if user is not None:
if user.is_active:
auth_login(request, user)
return HttpResponse('fine')
else:
return HttpResponse('inactive')
else:
return HttpResponse('bad')
[docs]def helpPage(request):
"""
Render help page.
"""
return render(
request, "modeldcc/help.html")
[docs]def termsPage(request):
"""
Render page with all controlled vocabulary terms as tables attached.
"""
statusesTable = StatusesTable(statuses.objects.all())
RequestConfig(
request, paginate={'per_page': 20}).configure(statusesTable)
biosampleTypesTable = BiosampleTypesTable(
biosample_types.objects.all())
RequestConfig(request,
paginate={'per_page': 20}).configure(biosampleTypesTable)
anatomicalTermsTable = AnatomicalTermsTable(
anatomical_terms.objects.all())
RequestConfig(
request, paginate={'per_page': 20}).configure(
anatomicalTermsTable)
labsTable = LabsTable(labs.objects.all())
RequestConfig(
request, paginate={'per_page': 20}).configure(labsTable)
geneticBackgroundsTable = GeneticBackgroundsTable(
genetic_backgrounds.objects.all())
RequestConfig(
request, paginate={'per_page': 20}).configure(
geneticBackgroundsTable)
developmentalStagesTable = DevelopmentalStagesTable(
developmental_stages.objects.all())
RequestConfig(
request, paginate={'per_page': 20}).configure(
developmentalStagesTable)
sexesTable = SexesTable(sexes.objects.all())
RequestConfig(request, paginate={'per_page': 20}).configure(
sexesTable)
assayTypesTable = AssayTypesTable(assay_types.objects.all())
RequestConfig(
request, paginate={'per_page': 20}).configure(assayTypesTable)
platformsTable = PlatformsTable(platforms.objects.all())
RequestConfig(
request, paginate={'per_page': 20}).configure(platformsTable)
instrumentsTable = InstrumentsTable(instruments.objects.all())
RequestConfig(
request, paginate={'per_page': 20}).configure(instrumentsTable)
strandModesTable = StrandModesTable(strand_modes.objects.all())
RequestConfig(
request, paginate={'per_page': 20}).configure(strandModesTable)
return render(
request, "modeldcc/terms.html", {
"statusesTable": statusesTable,
"biosampleTypesTable": biosampleTypesTable,
"anatomicalTermsTable": anatomicalTermsTable,
"labsTable": labsTable,
"geneticBackgroundsTable": geneticBackgroundsTable,
"developmentalStagesTable": developmentalStagesTable,
"sexesTable": sexesTable, "assayTypesTable": assayTypesTable,
"platformsTable": platformsTable,
"instrumentsTable": instrumentsTable,
"strandModesTable": strandModesTable})
[docs]@login_required
@user_passes_test(is_member)
def createindexJSON(request):
"""
Create a json file with data of assays and labs in the currently database.
Used for the plot in _index.js
"""
# assay2labs_json = cache.get('assay2labs.json')
# if not assay2labs_json:
sizes = []
sequences = []
stages = {'total_size': 0, 'stages': []}
assay_types = []
assay_types = AppliedAssay.objects.exclude(
biosamplereplicate__biosample__series__status=0).values('assay__assay_type__name').distinct()
for assay_type in assay_types:
tmp_sizes = {'name': assay_type['assay__assay_type__name']}
tmp_sequences = {'name': assay_type['assay__assay_type__name']}
labs = AppliedAssay.objects.exclude(biosamplereplicate__biosample__series__status=0).values(
'assay__assay_lab__name').distinct()
for lab in labs:
lab_name = lab['assay__assay_lab__name']
tmp_sizes[lab_name] = 0
data = Data.objects.exclude(
sequencing__technicalreplicate__appliedassay__biosamplereplicate__biosample__series__status=0)
data = data.filter(sequencing__technicalreplicate__appliedassay__assay__assay_type__name=assay_type[
'assay__assay_type__name'])
data = data.filter(sequencing__technicalreplicate__appliedassay__assay__assay_lab__name=lab[
'assay__assay_lab__name'])
data = data.filter(file_type__name='FASTQ')
for datum in data:
tmp_sizes[lab_name] += datum.primary_file.size
if datum.secondary_file:
tmp_sizes[lab_name] += datum.secondary_file.size
tmp_sizes[lab_name] = tmp_sizes[lab_name] / float(pow(2, 30))
tmp_sequences[lab_name] = 0
seqs = Sequencing.objects.exclude(
technicalreplicate__appliedassay__biosamplereplicate__biosample__series__status=0)
seqs = seqs.filter(technicalreplicate__appliedassay__assay__assay_type__name=assay_type[
'assay__assay_type__name'])
seqs = seqs.filter(technicalreplicate__appliedassay__assay__assay_lab__name=lab[
'assay__assay_lab__name']).values('sequencing_id')
tmp_sequences[lab_name] = len(seqs)
sizes.append(tmp_sizes)
sequences.append(tmp_sequences)
# cache.set('assay2labs.json', assay2labs_json, 86400)
with open(os.path.join(settings.MEDIA_ROOT,'assay2labs.json'), 'w') as outfile:
json.dump({'sizes': sizes, 'sequences': sequences}, outfile)
# return HttpResponse('index json created')
[docs]def get_plot_sizeDevelopment():
"""
Generate and send a json file containing the data for the area plot to show
the cummulative development of the amount of data in the DCC over time.
"""
# sizeDevelopment_json = []
sizeDevelopment_json = cache.get('sizeDevelopment.json')
if not sizeDevelopment_json:
sizes = []
assay_types_names = assay_types.objects.all().values_list('name') # get all assay type names
assay_types_names = map(lambda x: str(x[0]), assay_types_names) # convert list typles into strings
data = Data.objects.exclude(
sequencing__technicalreplicate__appliedassay__biosamplereplicate__biosample__series__status=0).values(
'date_added','sequencing__technicalreplicate__appliedassay__assay__assay_type__name','primary_file','secondary_file')
tmp_sizes = {}
for datum in data:
date = datum['date_added'].date().isoformat() #convert date into 'YYYY-MM-DD'
#create a ney key with empty values if date is not yet in dict
if date not in tmp_sizes:
tmp_sizes[date] = {k:0 for k in assay_types_names}
assay_name = str(datum['sequencing__technicalreplicate__appliedassay__assay__assay_type__name'])
file_size = 0
file_size += os.path.getsize(os.path.join(settings.MEDIA_ROOT,datum['primary_file']))
if str(datum['secondary_file']) is not '':
file_size += os.path.getsize(os.path.join(settings.MEDIA_ROOT,datum['secondary_file']))
tmp_sizes[date][assay_name] += file_size
for k,v in tmp_sizes.iteritems():
v['date'] = str(k)
sizes.append(v)
sizeDevelopment_json = simplejson.dumps(
{'sizes': sizes})
cache.set('sizeDevelopment.json', sizeDevelopment_json, 86400)
return sizeDevelopment_json
[docs]def get_plot_stages2assay():
"""
generate and send a json file contains a nested list of the amount of data
per lab per assay. Used to render the bar graph on the index page.
"""
# stages2labs_json =[]
stages2labs_json = cache.get('stages2labs.json')
if not stages2labs_json:
data = []
stages = Biosample.objects.exclude(series__status=0)\
.exclude(stage__name__isnull=True).values('stage__name')\
.distinct()
assay_types = {'Chromatin': ['DNAse-seq', 'MNase-seq', 'ChIP-seq', 'ATAC-seq'],
'Methylation': ['TAB-seq', 'MethylCap-seq', 'MethylC-seq', 'MeDIP-seq', 'BS-seq', 'RRBS'],
'RNA': ['SAPAS-seq','Ribo-seq', 'CAGE-seq','short-RNA-seq', 'RNA-seq', '3P-seq']}
assay_types_filtered = deepcopy(assay_types)
for stage in stages:
counts = AppliedAssay.objects.exclude(
biosamplereplicate__biosample__series__status=0)
counts = counts.filter(
biosamplereplicate__biosample__stage__name=stage['stage__name'])
tmp_data = {"stage": stage['stage__name']}
for types, subtypes in assay_types.iteritems():
for subtype in subtypes:
num_obj = AppliedAssay.objects.exclude(
biosamplereplicate__biosample__series__status=0)\
.filter(assay__assay_type__name=subtype).count()
if num_obj == 0:
if subtype in assay_types_filtered[types]:
assay_types_filtered[types].remove(subtype)
continue
subtype_counts = counts.filter(
assay__assay_type__name=subtype)
tmp_data[subtype] = len(subtype_counts)
data.append(tmp_data)
stages2labs_json = json.dumps(
{'data': data, 'assay_types': assay_types_filtered, })
cache.set('stages2labs.json', stages2labs_json, 86400)
return stages2labs_json
[docs]def get_plot_stages2chip():
"""
Generate and send a json file containing the stages and targets for the
ChiP-seq data, used in the bar plot on the visualzations page
"""
stages2chip_json = cache.get('stages2chip.json')
if not stages2chip_json:
data =[]
stages = Biosample.objects.filter(series__status = 1)\
.exclude(stage__name__isnull=True).values('stage__name')\
.distinct()
chips = Assay.objects.filter(assay_type__name='ChIP-seq')
targets = chips.exclude(assay_appliedassay__biosamplereplicate__biosample__series__status=0)\
.values_list('target', flat=True).distinct()
for stage in stages:
counts = AppliedAssay.objects.filter(assay__assay_type__name = 'ChIP-seq')
counts = counts.exclude(biosamplereplicate__biosample__series__status= 0)
counts = counts.filter(biosamplereplicate__biosample__stage__name = stage['stage__name'])
tmp_data ={"stage":stage['stage__name']}
for target in targets:
type_counts = counts.filter(assay__target = target)
tmp_data[target] = len(type_counts)
data.append(tmp_data)
stages2chip_json = simplejson.dumps({'data':data, 'assay_types':{'target':list(targets)},})
cache.set('stages2chip.json', stages2chip_json, 86400)
return stages2chip_json
def humanize_sizeof(num, suffix='B'):
num /= 2 ^ 30
return "%.2f%s%s" % (num, ' G', suffix)
def humanize_time(time):
day_unit = "d"
hour_unit = "h"
if "-" in time:
return time + hour_unit
if time == "adult":
return time
if not time or time == " ":
return "no value"
time = time.replace("hpf","")
time = time.replace("h","")
time = time.replace(",",".")
time = float(time)
if time>=24:
days = int(time/24)
hours = time % 24
if hours ==0:
return str(days) + day_unit
else:
if hours.is_integer():
hours = int(hours)
return str(days) + day_unit + " " + str(hours) + hour_unit
else:
if time.is_integer():
time = int(time)
return str(time)+hour_unit
def addFacet(fields, data):
tmp_facet = {}
for field in fields:
tmp_facet[field] = data.values(field).annotate(
count=Count(field, distinct=True))
return tmp_facet
# @login_required
# @user_passes_test(is_member)
[docs]def visualization(request):
"""
Render the visualization page and send the data files required for the
visualzations along with it.
"""
plot_data = {
'assays': get_plot_stages2assay(),
'chip': get_plot_stages2chip(),
'sizeDevelopment': get_plot_sizeDevelopment()
}
return render(request, 'modeldcc/visualization.html', {'plot_data': plot_data})
[docs]@login_required
def CombinedView(request):
"""
A deprecated table view, to show all the data for each section.
"""
if request.user.is_authenticated():
series_data = Series.objects.filter(
Q(user=request.user) |
~Q(status=0))
seriesTable = SeriesTable(series_data, order_by="-date_added")
biosample_data = Biosample.objects.filter(
Q(series__user=request.user) |
~Q(series__status=0))
else:
series_data = Series.objects.filter(~Q(status=0))
biosample_data = Biosample.objects.filter(
~Q(series__status=0)).only('id')
facet = {}
series_fields = ['series_type', 'status__name']
facet['series'] = addFacet(series_fields, series_data)
biosampleTable = BioTable(biosample_data, order_by="-biosample_id")
biosample_fields = ['biosample_type__name', 'biosample_lab__name',
'genetic_background__name', 'stage__name',
'anatomical_term__name', 'treatment',
'post_fertilization', 'mutation_description']
facet['biosample'] = addFacet(biosample_fields, biosample_data)
assay_data = Assay.objects.exclude(
assay_appliedassay__biosamplereplicate__biosample__series__status=0)
assay_data = assay_data.filter(assay_appliedassay__isnull=False).distinct()
# assay_data.exclude(assay_appliedassay__biosamplereplicate__biosample__series__series_id__isnull=True)
# assay_data = assay_dataexclude(assay_appliedassay__isnull=True)
assayTable = AssayTable(assay_data)
assay_fields = ['assay_type__name', 'assay_lab__name', 'target']
facet['assay'] = addFacet(assay_fields, assay_data)
return render(
request, "modeldcc/viewCompleteTab.html", {
"seriesTable": seriesTable, "biosampleTable": biosampleTable,
"assayTable": assayTable, "facet": facet,
})
# @login_required
# @user_passes_test(is_member)
@ensure_csrf_cookie
def dataExport(request):
return render(request, "modeldcc/dataExport.html")
[docs]@login_required
@user_passes_test(is_member)
# @cache_page(3600)
def createDataExportJSON(request):
"""
Create and save a json file with all available data sets and their metadata. Used in
the react dataExport app.
"""
series_list = []
flat_data = []
serieses = Series.objects.exclude(
status=0).exclude(is_public=False).prefetch_related('series_biosample')
for series in serieses:
biosample_list = []
lab_list = []
biosamples = series.series_biosample.all()
for biosample in biosamples:
lab_list.append(
{'biosample_lab': biosample.biosample_lab.name})
anatomical_term = 'no value'
if biosample.anatomical_term is not None:
anatomical_term = biosample.anatomical_term.name
stage = 'no value'
if biosample.stage is not None:
stage = biosample.stage.name
biosmaple_controlled_by = 'no value'
if biosample.controlled_by is not None:
biosmaple_controlled_by = biosample.controlled_by.biosample_id
mutation_description = 'no value'
if biosample.mutation_description is not ' ':
mutation_description = biosample.mutation_description
treatment = 'no value'
if biosample.treatment is not ' ':
treatment = biosample.treatment
biosamplereplicates = BiosampleReplicate.objects\
.filter(biosample=biosample.id).prefetch_related('biorep_appliedassay__assay__assay_type')
for biosamplereplicate in biosamplereplicates:
biosamplereplicate_item = {'applied_assay': []}
for appliedassay in biosamplereplicate.biorep_appliedassay.all():
assay_target = 'no value'
if appliedassay.assay.target is not None:
assay_target = appliedassay.assay.target
library_prep = 'no value'
if appliedassay.assay.library_prep is not None:
library_prep = appliedassay.assay.library_prep
appliedassay_controlled_by = 'no value'
if appliedassay.controlled_by is not None:
appliedassay_controlled_by = appliedassay.controlled_by.appliedassay_id
technicalreplicates = TechnicalReplicate.objects\
.filter(appliedassay=appliedassay.id)\
.prefetch_related('appliedassay_technicalreplicate__sequenced_by__sequencing_data')
for technicalreplicate in appliedassay.appliedassay_technicalreplicate.all():
for sequencing in technicalreplicate.sequenced_by.all():
if sequencing.paired_end:
paired_mode = "paired end"
else:
paired_mode = "single end"
strand_mode = 'no value'
if sequencing.strand_mode is not None:
strand_mode = sequencing.strand_mode.name
for data in sequencing.sequencing_data.all():
secondary_file = {
'name' : 'no value',
'size' : 'no value',
'url' : 'no value'
}
if data.secondary_file and hasattr(data.secondary_file, 'name'):
secondary_url = data.secondary_file.url
secondary_url = get_current_site(request).domain+ \
os.path.join(settings.MEDIA_URL,'annotated_files',secondary_url[7:])
secondary_file = {
'name': os.path.basename(data.secondary_file.name),
'size': data.secondary_file.size,
'url': secondary_url}
mapped_genome = 'no value'
if data.mapped_genome is not None:
mapped_genome = data.mapped_genome.name
version = 'no value'
if data.version is not None:
version = data.version.name
sha1 = 'no value'
if data.sha1 is not None:
sha1 = data.sha1
primary_url = data.primary_file.url
primary_url = get_current_site(request).domain + \
os.path.join(settings.MEDIA_URL,'annotated_files',primary_url[7:])
flat_data.append({'series_title': series.title,
'series_type': series.series_type,
'series_description': series.description,
'series_id': series.series_id,
'series_status': series.status.name,
'publication': series.publication,
'sra_geo_id': series.sra_geo_id,
'biosample_id': biosample.biosample_id,
'biosample_type': biosample.biosample_type.name,
'background': biosample.genetic_background.name,
'biosample_lab': biosample.biosample_lab.name,
'anatomical_term': anatomical_term,
'stage': stage,
'biosample_controlled_by': biosmaple_controlled_by,
'post_fertilization': humanize_time(biosample.post_fertilization),
'treatment': biosample.treatment,
'biosample_description': biosample.description,
'source': biosample.source,
'mutation_description': mutation_description,
'biosamplereplicate_id': biosamplereplicate.biosamplereplicate_id,
'assay_id': appliedassay.assay.assay_id,
'assay_type': appliedassay.assay.assay_type.name,
'assay_description': appliedassay.assay.description,
'assay_lab': appliedassay.assay.assay_lab.name,
'assay_target': assay_target,
'library_prep': library_prep,
'appliedassay_id': appliedassay.appliedassay_id,
"appliedassay_controlled_by" : appliedassay_controlled_by,
'technicalreplicate_id' : technicalreplicate.technicalreplicate_id,
'sequencing_id': sequencing.sequencing_id,
'sequencing_lab': sequencing.sequencing_lab.name,
'instrument': sequencing.instrument.name,
'date': sequencing.sequencing_date.isoformat(),
'strand_mode': strand_mode,
'max_read_length': sequencing.max_read_length,
'chemistry_version': sequencing.chemistry_version,
'platform': sequencing.platform.name,
'paired_end':paired_mode,
'data_id': data.data_id,
'primary_file_name': os.path.basename(data.primary_file.name),
'primary_file_size': data.primary_file.size,
'primary_file_url': primary_url,
'sha1': sha1,
'secondary_file_name':secondary_file['name'],
'secondary_file_size':secondary_file['size'],
'secondary_file_url':secondary_file['url'],
'file_type': data.file_type.name,
'mapped_genome': mapped_genome,
'command': data.command.split('|')[0],
'version': version,
})
filterKeys = [
'series_title',
'series_type',
'series_description',
'series_id',
'series_status',
'pulication',
'sra_geo_id',
'biosample_type',
'background',
'biosample_lab',
'anatomical_term',
'stage',
'controlled_by',
'post_fertilization',
'treatment',
'biosample_description',
'source',
'mutation_description',
'biosample_id',
'biosamplereplicate_id',
'assay_id',
'assay_type',
'assay_description',
'assay_lab',
'assay_target',
'technicalreplicate_id',
'appliedassay_id',
'sequencing_id',
'sequencing_lab',
'instrument',
'date',
'strand_mode',
'max_read_length',
'chemistry_version',
'platform',
'data_id',
'secondary_file',
'file_type',
'command',
'derived_from',
'version',
'mapped_genome'
]
flat_data = {'data': flat_data, 'filterKeys': filterKeys}
dataExport_json = json.dumps(flat_data)
with open(os.path.join(settings.MEDIA_ROOT, 'dataExport.json'), 'w') as outfile:
json.dump(flat_data, outfile)
# return HttpResponse('data export json created')
# @login_required
# @user_passes_test(is_member)
[docs]def filePaths(request):
"""
Render a list of the available files to help during a file upload, when
files are in the ``UPLOAD_PATH`` directory.
"""
res = []
for root, dirs, files in os.walk(UPLOAD_PATH):
for file in files:
path = re.sub('^' + UPLOAD_PATH, '', os.path.join(root, file))
lab = path.split('/')[0]
if lab == path:
continue
checksum = ''
res.append({'path': path, 'lab': lab, 'checksum': checksum})
return render(request, 'modeldcc/filepaths.html',
{'dirs': res})
# detail views
[docs]def hideEmptyColumns(data, table):
"""
Helper function to hide otherwise empty columns in the table views in the
detail pages.
"""
for field_name, column_obj in table.base_columns.iteritems():
is_blank = True
for datum in data:
if field_name == 'short_desc':
is_blank = False
elif getattr(datum, field_name):
is_blank = False
if is_blank:
column_obj.visible = False
[docs]def SeriesChildTable(request, series_id):
"""
Returns a list of biosamples and assays for the given series_id to be used
in the table on the seriesDetail page.
"""
series = get_object_or_404(Series, pk=series_id)
biosamp = []
assays = []
biosample_list = []
for biosamples in series.series_biosample.all().only('id'):
biosample = {'type': 'biosample', 'instance': biosamples,
'biosample_children': [], 'table': {}}
biosample_data = Biosample.objects.filter(
biosample_id=biosamples.biosample_id)
biosample['table'] = BioTable(biosample_data, orderable=False)
RequestConfig(request, paginate={'per_page': 20}).configure(
biosample['table'])
hideEmptyColumns(biosample_data, biosample['table'])
biosamp.append(biosamples)
for biosamplereplicates in BiosampleReplicate.objects.filter(biosample__biosample_id=biosamples.biosample_id):
biosamplereplicate = {'type': 'biosamplereplicate',
'instance': biosamplereplicates, 'biosamplereplicate_children': []}
for appliedassays in AppliedAssay.objects.filter(biosamplereplicate__biosamplereplicate_id=biosamplereplicates.biosamplereplicate_id).order_by("assay__assay_type"):
appliedassay = {
'type': 'appliedassay', 'instance': appliedassays, 'appliedassay_children': []}
appliedassay_data = AppliedAssay.objects.filter(
appliedassay_id=appliedassays.appliedassay_id)
appliedassay['table'] = AppliedAssayTable(
appliedassay_data, orderable=False)
RequestConfig(request, paginate={'per_page': 20}).configure(
appliedassay['table'])
hideEmptyColumns(appliedassay_data, appliedassay['table'])
# assays.append(appliedassays.assay)
for technicalreplicates in TechnicalReplicate.objects.filter(appliedassay__appliedassay_id=appliedassays.appliedassay_id):
technicalreplicate = {
'type': 'technicalreplicate', 'instance': technicalreplicates, 'technicalreplicate_children': []}
for sequencings in Sequencing.objects.filter(technicalreplicate__technicalreplicate_id=technicalreplicates.technicalreplicate_id):
sequencing = {
'type': 'sequencing', 'instance': sequencings, 'sequencing_children': []}
sequencing_data = Sequencing.objects.filter(
sequencing_id=sequencings.sequencing_id)
sequencing['table'] = SequencingTable(
sequencing_data, orderable=False)
RequestConfig(request, paginate={'per_page': 20}).configure(
sequencing['table'])
hideEmptyColumns(sequencing_data, sequencing['table'])
for data in Data.objects.filter(sequencing__sequencing_id=sequencings.sequencing_id):
datum = {'type': 'data', 'instance': data}
sequencing['sequencing_children'].append(datum)
technicalreplicate[
'technicalreplicate_children'].append(sequencing)
appliedassay['appliedassay_children'].append(
technicalreplicate)
biosamplereplicate[
'biosamplereplicate_children'].append(appliedassay)
biosample['biosample_children'].append(biosamplereplicate)
biosample_list.append(biosample)
for appliedassays in AppliedAssay.objects.filter(biosamplereplicate__biosample__series_id=series_id).order_by("assay__assay_type"):
assays.append(appliedassays.assay)
assays = list(set(assays))
assays.sort(key=lambda x: x.assay_type.name)
biosamples = list(set(biosamp))
return (series, biosamples, assays, biosample_list)
# @login_required
[docs]def SeriesDetail(request, series_id):
"""
Render a page with series details to a given series_id, including related
items.
"""
series, biosamples, assays, biosample_list = SeriesChildTable(
request, series_id)
return render(
request, 'modeldcc/detailSeries.html',
{'series': series,
'biosamples': biosamples, 'assays': assays,
'biosample_list': biosample_list})
# @login_required
[docs]def BiosampleDetail(request, biosample_id):
"""
Render a page with biosample details to a given biosample_id, including related
items.
"""
biosample = get_object_or_404(Biosample, pk=biosample_id)
biosamplereplicates = []
appliedassays = []
assays = []
for biosamplereplicate in biosample.biosample_biorep.all():
biosamplereplicates.append(biosamplereplicate)
for biosamplereplicate in biosamplereplicates:
for appliedassay in biosamplereplicate.biorep_appliedassay.all():
appliedassays.append(appliedassay)
for appliedassay in appliedassays:
assays.append(appliedassay.assay)
return render(
request, 'modeldcc/detailBio.html',
{'biosample': biosample, 'assays': assays,
'biosamplereplicates': biosamplereplicates,
'appliedassays': appliedassays})
# @login_required
[docs]def AssayDetail(request, assay_id):
"""
Render a page with assay details to a given assay_id, including related
items.
"""
assay = get_object_or_404(Assay, pk=assay_id)
used_in_series = []
connected_series = []
if request.user.is_authenticated():
for appliedassay in assay.assay_appliedassay.filter(
Q(biosamplereplicate__biosample__series__user=request.user) |
~Q(biosamplereplicate__biosample__series__status=0)):
used_in_series.append(
appliedassay.biosamplereplicate.biosample.series.series_id)
else:
for appliedassay in assay.assay_appliedassay.filter(
~Q(biosamplereplicate__biosample__series__status=0)):
used_in_series.append(
appliedassay.biosamplereplicate.biosample.series.series_id)
used_in_series = list(set(used_in_series))
for series in used_in_series:
connected_series.append(Series.objects.get(series_id=series))
return render(request, 'modeldcc/detailAssay.html',
{'assay': assay, 'connected_series': connected_series})
# @login_required
[docs]def AppliedAssayDetail(request, appliedassay_id):
"""
Render a page with applied assay details to a given appliedassay_id, including related
items.
"""
applied_assay = get_object_or_404(AppliedAssay, pk=appliedassay_id)
return render(request, 'modeldcc/detailAppliedAssay.html',
{'applied_assay': applied_assay})
# @login_required
[docs]def SequencingDetail(request, sequencing_id):
"""
Render a page with sequencing details to a given sequencing_id, including related
items.
"""
sequencing = get_object_or_404(Sequencing, pk=sequencing_id)
used_in_series = []
connected_series = []
user = sequencing.user
status = sequencing.technicalreplicate.appliedassay.biosamplereplicate.biosample.series.status_id
connected_series = [] if status == \
statuses.objects.get(name="initial entry").id and \
request.user != user else \
[sequencing.technicalreplicate.appliedassay.biosamplereplicate.biosample.series, ]
return render(
request, 'modeldcc/detailSequencing.html',
{'sequencing': sequencing, 'connected_series': connected_series})
# @login_required
[docs]def DataDetail(request, data_id):
"""
Render a page with data details to a given data_id, including related
items.
"""
data = get_object_or_404(Data, pk=data_id)
user = data.user
status = data.sequencing.technicalreplicate.appliedassay.biosamplereplicate.biosample.series.status_id
connected_series = [] if status == \
statuses.objects.get(name="initial entry").id and \
request.user != user else \
[data.sequencing.technicalreplicate.appliedassay.biosamplereplicate.biosample.series, ]
return render(
request, 'modeldcc/detailData.html',
{'data': data, 'connected_series': connected_series})
[docs]@login_required
@user_passes_test(is_member)
def addLab(request):
"""
Add a lab to the DCC system.
"""
table = LabsTable(
labs.objects.all(), order_by="name")
error = None
if request.method == 'POST':
if not request.POST['lab_name'] == '':
try:
lab_exists = labs.objects.get(
name=request.POST['lab_name'])
error = "Lab name exists already"
except labs.DoesNotExist:
newLab = labs(
name=request.POST['lab_name'],
lab_pi=request.POST['lab_pi'],
lab_institution=request.POST['lab_institution'],
lab_country=request.POST['lab_country']
)
newLab.save()
else:
error = "Please provide at least a lab name"
return render(
request, 'modeldcc/addLab.html',
{"table": table, "error": error})
[docs]@login_required
@user_passes_test(is_member)
def delete(request):
"""
Mark a series as deleted, i.e. reset it's status to ``INITIAL_UPLOAD_STATUS``.
"""
series = Series.objects.get(series_id=request.POST['id'])
series.status = statuses.objects.get(
name=INITIAL_UPLOAD_STATUS)
series.save()
payload = {'success': True}
createDataExportJSON(request)
createindexJSON(request)
return JsonResponse(json.dumps(payload), safe=False)
[docs]@login_required
@user_passes_test(is_member)
def deleteFiles(request):
"""
Delete files from the DCC via an ajax request with the corresponding dataids.
"""
try:
data = json.loads(request.body)['id']
except:
data = json.loads(request.POST.get("id"))
for datum in data:
datum_object = Data.objects.get(data_id=datum['dataid'])
directory = os.path.dirname(datum_object.primary_file.path)
datum_object.delete()
if os.path.isdir(directory) and not os.listdir(directory):
os.rmdir(directory)
payload = {'success': True}
createDataExportJSON(request)
return JsonResponse(json.dumps(payload), safe=False)
[docs]@login_required
@user_passes_test(is_member)
def deleteFilesByPath(request):
"""
Delete files from the systme via an ajax request with the corresponding
filepath.
"""
filepaths = json.loads(request.body)['id']
payload = {'success': True}
for fpath in filepaths:
fp = os.path.join(settings.MEDIA_ROOT_DCC, fpath['path'])
if os.path.exists(fp):
os.remove(fp)
directory = os.path.dirname(fp)
if not os.listdir(directory):
os.rmdir(directory)
else:
print("The file " + fpath['path'] + " does not exist")
payload = {'success': False}
return JsonResponse(json.dumps(payload), safe=False)
[docs]@login_required
@user_passes_test(is_member)
def addFreezeTag(request):
"""
Update the version field for a given data id, previous version numbers are
concatenated with the new one.
"""
data = json.loads(request.body)['id']
for datum in data:
datum_object = Data.objects.get(data_id=datum['dataid'])
if datum_object.version_id is not None:
datum_object.version = Versions.objects.get(name=datum_object.version.name + ", " + datum['freeze_tag'])
else:
datum_object.version = Versions.objects.get(name=datum['freeze_tag'])
datum_object.save()
payload = {'success': True}
createDataExportJSON(request)
return JsonResponse(json.dumps(payload), safe=False)
[docs]@login_required
@user_passes_test(lambda u: u.is_superuser)
@user_passes_test(is_member)
def importDNANexus(request):
"""
Render the interface to upload analyzed data (originally via DNANexus).
"""
return render(request, 'modeldcc/importDNANexus.html')
@login_required
@user_passes_test(lambda u: u.is_superuser)
@user_passes_test(is_member)
def addDataPage(request):
return render(request, 'modeldcc/addDataPage.html')
# @login_required
# @user_passes_test(is_member)
[docs]def csv_export(request):
"""
Create and send a csv file containing all the metadata for a given series_id
and it's connected sections.
"""
series_id = request.GET.get('series_id')
data = Data.objects.filter(
sequencing__technicalreplicate__appliedassay__biosamplereplicate__biosample__series__series_id=series_id)
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="%s.csv"' % series_id
writer = csv.writer(response)
models = [Series, Biosample, BiosampleReplicate, Assay,
AppliedAssay, Sequencing, TechnicalReplicate, Data]
headers = []
for model in models:
# Write headers to CSV file
for field in model._meta.fields:
if field.name == "id":
continue
headers.append(model.__name__ + '__' + field.name)
writer.writerow(headers)
# Write data to CSV file
for datum in data:
row = []
queryset = [datum,
datum.sequencing,
datum.sequencing.technicalreplicate,
datum.sequencing.technicalreplicate.appliedassay,
datum.sequencing.technicalreplicate.appliedassay.assay,
datum.sequencing.technicalreplicate.appliedassay.biosamplereplicate,
datum.sequencing.technicalreplicate.appliedassay.biosamplereplicate.biosample,
datum.sequencing.technicalreplicate.appliedassay.biosamplereplicate.biosample.series]
for field in headers:
if field in headers:
for obj in reversed(queryset):
className = obj.__class__.__name__
if className == field.split('__')[0]:
try:
header = field.split('__')[1]
val = getattr(obj, header)
if callable(val):
val = val()
# work around csv unicode limitation
if type(val) == unicode:
val = val.encode("utf-8")
if header.endswith('_file') and str(val):
val = get_current_site(request).domain + '/files/annotated_files/' + \
str(val)
row.append(val)
except AttributeError:
continue
writer.writerow(row)
# Return CSV file to browser as download
return response
# @login_required
# @user_passes_test(is_member)
[docs]def getZippedFiles(request):
"""
Create and send a .tar archive of requested filepaths.
"""
# Files (local path) to put in the .zip
if request.method != 'GET':
raise Http404("Must be a GET request!")
data = request.GET.get("filepaths")
filenames = data.split(',')
# Folder name in ZIP archive which contains the above files
# E.g [thearchive.zip]/somefiles/file2.txt
tar_subdir = "dcc_export"
tar_name = "dcc_data_export.tar.gz"
s = StringIO.StringIO()
tmp_dir = os.path.join(settings.MEDIA_ROOT,'tmp')
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
tmp_tar = NamedTemporaryFile(suffix='.tar.gz',dir=tmp_dir)
# Open StringIO to grab in-memory ZIP contents
# s = StringIO.StringIO()
# The tar compressor
tar = tarfile.open(tmp_tar.name, mode='w:gz',)
for fpath in filenames:
media_fpath = os.path.join(settings.MEDIA_ROOT_DCC,fpath[7:])
tar.add(media_fpath,os.path.basename(fpath))
# Must close tar for all contents to be written
tar.close()
# Grab ZIP file from in-memory, make response with correct MIME-type
response = FileResponse(FileWrapper(tmp_tar), content_type = "application/x-gzip")
# ..and correct content-disposition
response['Content-Disposition'] = 'attachment; filename=%s' % tar_name
return response
# @login_required
# @user_passes_test(is_member)
[docs]def getURLs(request):
"""
Create and send a file, where each line contains the url to a file on the
server requested via their relative filepaths.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
data = json.loads(request.body)['filepaths']
urls = ''
for fpath in data:
urls = urls + get_current_site(request).domain + \
os.path.join(settings.MEDIA_URL,'annotated_files',fpath[7:]) + '\n'
file_name = "dcc_url_export.txt"
response = FileResponse(urls, content_type = "text/plain")
response['Content-Disposition'] = 'attachment; filename=%s' % file_name
return response
[docs]@login_required
@user_passes_test(is_member)
def addSeries(request):
"""
Add a series object to the DCC and return a list of available assay and labs
for the web interface.
"""
available_assays_tmp = []
for assay in Assay.objects.all():
available_assays_tmp.append({
'assay_id':assay.assay_id,
'assay_lab':assay.assay_lab.name,
'assay_type':assay.assay_type.name,
'library_prep':assay.library_prep,
'target':assay.target,
'description':assay.description
})
available_assays = simplejson.dumps(available_assays_tmp)
labs_tmp = []
for lab in labs.objects.all():
labs_tmp.append({
'lab_id':lab.lab_id,
'name':lab.name,
'lab_pi':lab.lab_pi,
'lab_country':lab.lab_country,
'lab_institution':lab.lab_institution
})
labs_tmp = sorted(labs_tmp, key=lambda k: k['name'], reverse=True)
labs_list = simplejson.dumps(labs_tmp)
if request.method == 'POST':
if request.POST.get('delete'):
obj.delete()
if not request.POST:
json_series = request.body
else:
json_series = request.POST.get('jsonData')
series_data = json.loads(json_series)
series_field = series_data[0]['fields']
this_series = Series.objects.filter(
series_id=series_field['series_id'])
if this_series:
this_series.update(
title=series_field['series_title'],
publication=series_field['series_publication'],
series_type=series_field['series_type'],
description=series_field['series_description'],
is_public=series_field['series_is_public'] == 'True',
sra_geo_id=series_field['series_geo_id'],
status=statuses.objects.get(
name=this_series.first().status.name),
user=request.user,
date_edited=datetime.now()
)
series_response = {'series_id': series_field['series_id']}
return JsonResponse(series_response)
else:
# take the "fields" from json data and create Series instance
saveSeries = Series(
title=series_field['series_title'],
publication=series_field['series_publication'],
series_type=series_field['series_type'],
description=series_field['series_description'],
is_public=series_field['series_is_public'] == 'True',
sra_geo_id=series_field['series_geo_id'],
status=statuses.objects.get(
name=INITIAL_UPLOAD_STATUS),
user=request.user
)
saveSeries.save()
series_id = saveSeries.series_id
series_response = {'series_id': series_id}
# clean caches
cache.delete('assay2labs.json')
cache.delete('stages2labs.json')
cache.delete('dataExport.json')
cache.delete('stages2chip.json')
return JsonResponse(series_response)
return render(request, 'modeldcc/addSeries.html', {
'available_assays': available_assays, 'labs_list': labs_list}
)
[docs]@login_required
@user_passes_test(is_member)
def addBiosample(request):
"""
Add biosample and corresponding biosmaple replicate objects to the DCC and
return an array of their ids for the webinterface.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
json_biosample = request.body
bio_data = json.loads(json_biosample)
# taking the current series_id
part_of = Series.objects.get(
series_id=bio_data[0]['fields']['series_id'])
for i in bio_data:
# save biosamples
bio_field = i['fields']
this_biosample = Biosample.objects.filter(
biosample_id=bio_field['biosample_id'])
# control = Biosample.objects.filter(biosample_id = bio_field[])
if this_biosample:
this_biosample.update(
series=part_of,
biosample_lab=bio_field['biosample_lab'],
biosample_type=bio_field['biosample_sample_type'],
genetic_background=bio_field[
'biosample_genetic_background'],
anatomical_term=bio_field[
'biosample_anatomical_origin'],
mutation_description=bio_field[
'biosample_mutation_description'],
stage=bio_field['biosample_developmental_stage'],
treatment=bio_field['biosample_treatment'],
cell_line_type=bio_field[
'biosample_cell_line_type'],
description=bio_field['biosample_description'],
post_fertilization=bio_field[
'biosample_post_fertilization'],
source=bio_field['biosample_source'],
sex=bio_field['biosample_sex'],
controlled_by=None,
user=request.user
)
series_response = {'series_id': series_field['series_id']}
return JsonResponse(series_response)
else:
# take the "fields" from json data and create a Biosample instance
try:
anatomical_term_tmp = anatomical_terms.objects.get(
name=bio_field['biosample_anatomical_term'])
except anatomical_terms.DoesNotExist:
anatomical_term_tmp = None
try:
sex_tmp = sexes.objects.get(
name=bio_field['biosample_sex'])
except sexes.DoesNotExist:
sex_tmp = None
try:
stage_tmp = developmental_stages.objects.get(
name=bio_field['biosample_developmental_stage'])
except developmental_stages.DoesNotExist:
stage_tmp = None
if bio_field['control'] != None:
try:
controlled_by_tmp = Biosample.objects.filter(
series=part_of)[bio_field['control']]
except Biosample.DoesNotExist:
controlled_by_tmp = None
else:
controlled_by_tmp = None
saveBiosample = Biosample(
series=part_of,
biosample_lab=labs.objects.get(
name=bio_field['biosample_lab']),
biosample_type=biosample_types.objects.get(
name=bio_field['biosample_sample_type']),
genetic_background=genetic_backgrounds.objects.get(
name=bio_field['biosample_genetic_background']),
anatomical_term=anatomical_term_tmp,
mutation_description=bio_field[
'biosample_mutation_description'],
stage=stage_tmp,
treatment=bio_field['biosample_treatment'],
cell_line_type=bio_field[
'biosample_cell_line_type'],
description=bio_field['biosample_description'],
post_fertilization=bio_field[
'biosample_post_fertilization'],
source=bio_field['biosample_source'],
sex=sex_tmp,
controlled_by=controlled_by_tmp,
user=request.user
)
saveBiosample.save()
num_replicates = bio_field['num_replicates']
for rep in range(num_replicates):
saveReplicate = BiosampleReplicate(
biosample=saveBiosample,
user=request.user
)
saveReplicate.save()
# filter all biosamples replicates to get their ids
this_biosamplereplicates = BiosampleReplicate.objects.filter(
biosample__series=part_of).order_by('biosamplereplicate_id')
biosamplereplicates = serializers.serialize(
'json', this_biosamplereplicates)
biosamplereplicates = json.loads(biosamplereplicates)
biosamplereplicate_response = []
for n in biosamplereplicates:
i = n['fields']
individual_id = {'biosamplereplicate_id':
i['biosamplereplicate_id']}
biosamplereplicate_response.append(individual_id)
return JsonResponse(biosamplereplicate_response, safe=False)
[docs]@login_required
@user_passes_test(is_member)
def addAssay(request):
"""
Add assay objects to the DCC and
return an array of their ids for the webinterface.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
json_assay = request.body
assay_data = json.loads(json_assay)
this_assays = []
for i in assay_data:
assay_field = i['fields']
input_desc = "".join(
assay_field['assay_description'].lower().split())
# filter whether a particular assay type exists and
# from the same lab
if assay_field['target'] == '':
if assay_field['lib_prep'] == '':
filtered_assay = Assay.objects.filter(
assay_type=assay_types.objects.get(
name=assay_field['assay_type'])).filter(
assay_lab=labs.objects.get(
name=assay_field['assay_lab']))
else:
filtered_assay = Assay.objects.filter(
assay_type=assay_types.objects.get(
name=assay_field['assay_type'])).filter(
assay_lab=labs.objects.get(
name=assay_field['assay_lab'])).filter(
target=assay_field['lib_prep'])
else:
filtered_assay = Assay.objects.filter(
assay_type=assay_types.objects.get(
name=assay_field['assay_type'])).filter(
assay_lab=labs.objects.get(
name=assay_field['assay_lab'])).filter(
target=assay_field['target']
)
if filtered_assay:
for each_assay in filtered_assay:
# check if description doesn't exist
if input_desc != "".join(
each_assay.description.lower().split()):
# similar assay(s) with different description,
# then save
assay = Assay(
assay_type=assay_types.objects.get(
name=assay_field['assay_type']),
assay_lab=labs.objects.get(
name=assay_field['assay_lab']),
description=assay_field[
'assay_description'],
target=assay_field['target'],
library_prep=assay_field['lib_prep'],
user=request.user
)
assay.save()
individual_id = {'assay_id': assay.assay_id}
this_assays.append(individual_id)
elif input_desc == "".join(
each_assay.description.lower().split()):
# exactly same assay, just take the id
individual_id = {'assay_id': each_assay.assay_id}
this_assays.append(individual_id)
break
else:
# new assay (either new type from diff lab or new lab
# with diff type of assay)
assay = Assay(
assay_type=assay_types.objects.get(
name=assay_field['assay_type']),
assay_lab=labs.objects.get(
name=assay_field['assay_lab']),
description=assay_field['assay_description'],
target=assay_field['target'],
library_prep=assay_field['lib_prep'],
user=request.user
)
assay.save()
individual_id = {'assay_id': assay.assay_id}
this_assays.append(individual_id)
return JsonResponse(this_assays, safe=False)
[docs]@login_required
@user_passes_test(is_member)
def addAppliedAssay(request):
"""
Add applied assay objects to the DCC and
return an array of their ids for the webinterface.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
json_applied_assay = request.body
applied_assay_data = json.loads(json_applied_assay)
this_applied_assays = []
for i in applied_assay_data:
# linker table between biosample and assay (libraries)
applied_assay_field = i['fields']
previous_each_assay = None
for each_assay in applied_assay_field[
'appliedAssay_id'].split(" ")[::-1]:
control = None
if previous_each_assay is not None:
if "_control" in previous_each_assay:
control = applied_assay_instance_tmp
try:
applied_assay_instance = AppliedAssay.objects.get(
biosamplereplicate=BiosampleReplicate.objects.get(
biosamplereplicate_id=applied_assay_field['biosamplereplicate_id']),
assay=Assay.objects.get(
assay_id=each_assay + 'AS'),
controlled_by=control,
user=request.user
)
except AppliedAssay.DoesNotExist:
applied_assay_instance = AppliedAssay(
biosamplereplicate=BiosampleReplicate.objects.get(
biosamplereplicate_id=applied_assay_field['biosamplereplicate_id']),
assay=Assay.objects.get(
assay_id=each_assay + 'AS'),
controlled_by=control,
user=request.user
)
applied_assay_instance.save()
applied_assay_instance_tmp = applied_assay_instance
previous_each_assay = each_assay
num_tech_replicates = applied_assay_field[
'appliedAssay_num_tech_replicates']
for rep in range(int(num_tech_replicates)):
saveTechReplicate = TechnicalReplicate(
appliedassay=applied_assay_instance,
user=request.user
)
saveTechReplicate.save()
individual_id = {
'technicalReplicate_id':
saveTechReplicate.technicalreplicate_id}
this_applied_assays.append(individual_id)
return JsonResponse(this_applied_assays, safe=False)
[docs]@login_required
@user_passes_test(is_member)
def addSequencing(request):
"""
Add sequencing objects to the DCC and
return an array of their ids for the webinterface.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
json_sequencing = request.body
seq_data = json.loads(json_sequencing)
this_sequencing = []
for i in seq_data:
# save sequencing
seq_field = i['fields']
part_of = TechnicalReplicate.objects.get(
technicalreplicate_id=seq_field['technicalReplicate_id'])
try:
lab_tmp = labs.objects.get(
name=seq_field['sequencing_lab'])
except labs.DoesNotExist:
raise
try:
platform_tmp = platforms.objects.get(
name=seq_field['sequencing_platform'])
except platforms.DoesNotExist:
raise
try:
instrument_tmp = instruments.objects.get(
name=seq_field['sequencing_instrument'])
except instruments.DoesNotExist:
raise
if 'sequencing_strand_mode' in seq_field:
try:
strand_mode_tmp = strand_modes.objects.get(
name=seq_field['sequencing_strand_mode'])
except strand_modes.DoesNotExist:
raise
else:
strand_mode_tmp = None
if not seq_field['sequencing_max_read_length']:
read_length_tmp = None
else:
read_length_tmp = int(seq_field[
'sequencing_max_read_length'])
saveSequencing = Sequencing(
technicalreplicate=part_of,
sequencing_lab=lab_tmp,
platform=platform_tmp,
instrument=instrument_tmp,
chemistry_version=seq_field['sequencing_chem_version'],
max_read_length=read_length_tmp,
strand_mode=strand_mode_tmp,
paired_end=seq_field[
'sequencing_paired_end'] == "paired end",
sequencing_date=seq_field['sequencing_date'],
user=request.user
)
saveSequencing.save()
individual_id = {
'sequencing_id': saveSequencing.sequencing_id}
this_sequencing.append(individual_id)
return JsonResponse(this_sequencing, safe=False)
# note this is also used in the batch upload
[docs]def get_file_object(data_url, data_local_path):
"""
Helper function to get a file either via an url or via a file path on the
server
"""
if data_url is not None:
http = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
# get URL file
file_r = http.urlopen(
'GET', data_url, preload_content=False)
local_fp = NamedTemporaryFile(delete=True,
dir=os.path.join(settings.MEDIA_ROOT,'tmp'))
while True:
data = file_r.read(1024)
if not data:
break
local_fp.write(data)
local_fp.flush()
file_r.release_conn()
file_str = "url:::" + data_url
elif data_local_path is not None:
if data_local_path:
if data_local_path.startswith('/'):
data_local_path = data_local_path[1:]
local_fp = open(os.path.join(UPLOAD_PATH, data_local_path.strip()))
file_str = "local_path:::" + data_local_path
else:
local_fp = None
file_str = None
else:
local_fp = None
file_str = None
return local_fp, file_str
[docs]@login_required
@user_passes_test(is_member)
def addData(request):
"""
Add data objects to the DCC, upload or move the files (via url or a
filepath), change the series status to ``COMPLETED_UPLOAD_STATUS``
and return the corresponding series id.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
json_data = request.body
data_data = json.loads(json_data)
for i in data_data:
# save data
data_field = i['fields']
part_of = Sequencing.objects.get(
sequencing_id=data_field['sequencing_id'])
try:
primary_file, primary_source = get_file_object(
data_field['data_primary_file'], None)
except:
try:
primary_file, primary_source = get_file_object(
None, data_field['data_primary_file'])
except IOError:
return HttpResponseNotFound(data_field['data_primary_file'] +
" is not a valid URL or file path on the server")
try:
secondary_file, secondary_source = get_file_object(
data_field['data_secondary_file'], None)
except:
try:
secondary_file, secondary_source = get_file_object(
None, data_field['data_secondary_file'])
except IOError:
return HttpResponseNotFound(data_field['data_secondary_file'] +
" is not a valid URL or file path on the server")
saveData = Data(
sequencing=part_of,
primary_file=File(primary_file),
secondary_file=File(secondary_file),
primary_source=primary_source,
secondary_source=secondary_source,
mapped_genome=None,
file_type=file_types.objects.get(name='FASTQ'),
command='',
user=request.user
)
saveData.save()
primary_file.close()
if secondary_file is not None:
secondary_file.close()
series_id = saveData.sequencing.technicalreplicate.appliedassay.\
biosamplereplicate.biosample.series.series_id
series_pk = saveData.sequencing.technicalreplicate.appliedassay.\
biosamplereplicate.biosample.series.id
series_ref = Series.objects.get(series_id=series_id)
series_ref.status = statuses.objects.get(
name=COMPLETED_UPLOAD_STATUS)
series_ref.save()
createDataExportJSON(request)
createindexJSON(request)
return JsonResponse(series_pk, safe=False)
[docs]@login_required
@user_passes_test(is_member)
def addAnalyzedData(request):
"""
Add and upload analyzed data objects to the DCC and
return a flag if they already exist and their data ids.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
json_data = request.body
data_field = json.loads(json_data)
if not data_field:
raise Http404("empty JSON")
this_data = []
if not os.path.exists(os.path.join(settings.MEDIA_ROOT,'log_files')):
os.makedirs(os.path.join(settings.MEDIA_ROOT,'log_files'))
log_file = open(os.path.join(settings.MEDIA_ROOT,'log_files','DNANexus.log'),"a+")
try:
datum = Data.objects.get(primary_source=data_field['primary_source'])
already_exists = True
except Data.MultipleObjectsReturned:
datum = Data.objects.filter(primary_source=data_field['primary_source'])[0]
already_exists = True
except Data.DoesNotExist:
derived_from = Data.objects.get(data_id=data_field['derived_from'])\
.sequencing.technicalreplicate.appliedassay
part_of = Data.objects.get(
data_id=data_field['derived_from']).sequencing
try:
primary_file, primary_source = get_file_object(
data_field['primary_file'], None)
except:
try:
primary_file, primary_source = get_file_object(
None, data_field['primary_file'])
except IOError:
return HttpResponseNotFound(data_field['primary_file'] +
" is not a valid URL or file path on the server")
if data_field['secondary_file'] != '':
try:
secondary_file, secondary_source = get_file_object(
data_field['secondary_file'], None)
except:
try:
secondary_file, secondary_source = get_file_object(
None, data_field['secondary_file'])
except IOError:
return HttpResponseNotFound(data_field['secondary_file'] +
" is not a valid URL or file path on the server")
else:
secondary_file, secondary_source = [None, None]
file_type = data_field['file_type']
mapped_genome = data_field['mapped_genome']
saveData = Data(
sequencing=part_of,
primary_file=File(primary_file),
secondary_file=File(secondary_file),
primary_source=data_field['primary_source'],
secondary_source=data_field['secondary_source'],
mapped_genome=mapped_genomes.objects.get(name=mapped_genome),
file_type=file_types.objects.get(name=file_type),
command=data_field['command'],
derived_from=derived_from,
user=request.user
)
saveData.save()
if data_field['derived_from_data'] != {}:
for data in data_field['derived_from_data']:
saveData.derived_from_data.add(Data.objects.get(data_id=data))
primary_file.close()
if secondary_file is not None:
secondary_file.close()
individual_id = {'dataid': saveData.data_id}
this_data.append(individual_id)
series_id = saveData.sequencing.technicalreplicate.appliedassay.\
biosamplereplicate.biosample.series.series_id
series_pk = saveData.sequencing.technicalreplicate.appliedassay.\
biosamplereplicate.biosample.series.id
series_ref = Series.objects.get(series_id=series_id)
series_ref.status = statuses.objects.get(
name=WITH_ASSOCIATED_FILES)
series_ref.save()
series, biosamples, assays, biosample_list = SeriesChildTable(
request, series_pk)
log_file.write("finished uploading "+data_field['primary_file']+ " and saved it as "+ saveData.data_id + "\n")
log_file.close()
datum = Data.objects.get(data_id=saveData.data_id)
already_exists = False
createDataExportJSON(request)
createindexJSON(request)
return render(
request, 'modeldcc/importDNANexusSuccess.html',
{'datum': datum,'already_exists': already_exists,
'uploaded_data_ids': simplejson.dumps(this_data)})
[docs]@login_required
@user_passes_test(is_member)
def updateFilePaths(request):
"""
Rename/move and move files to reflect changes in their annotations. Moves
them first to the new position with a `__tmp` attached to the file name.
:meth:`modeldcc.views.updateTemporaryFilePaths` converts the name to
the final form.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
json_data = request.body
json_data = json.loads(json_data)
if not json_data:
raise Http404("empty JSON")
this_data = []
if not os.path.exists(os.path.join(settings.MEDIA_ROOT,'log_files')):
os.makedirs(os.path.join(settings.MEDIA_ROOT,'log_files'))
log_file = open(os.path.join(settings.MEDIA_ROOT,'log_files','updatedFilePaths.log'),"a+")
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="updatedData.csv"'
writer = csv.writer(response)
writer.writerow(["old file path","new file path"])
for data_field in json_data:
try:
data = Data.objects.filter(sequencing__technicalreplicate__appliedassay__biosamplereplicate__biosample__series__series_id=data_field['series_id'])
except Data.DoesNotExist:
does_not_exist = True
return Http404("series does not exist")
for datum in data:
log_file.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") +": started updating "+ datum.data_id + "\n")
log_file.write("old file path: "+ datum.primary_file.path +"\n")
try:
file_name_end = datum.primary_file.name.split('/')[2].split('.',2)[2]
if ".__tmp" in file_name_end:
# file_name_end = file_name_end.replace('.__tmp','')
file_name_end = file_name_end
else:
file_name_end = file_name_end + ".__tmp"
new_file_name_end = '.'.join(map(str, (datum.sequencing.technicalreplicate.appliedassay.assay.assay_id,
datum.sequencing.sequencing_id,file_name_end)))
except IndexError:
file_name_end = datum.primary_file.name.split('/')[2]
if ".__tmp" in file_name_end:
# file_name_end = file_name_end.replace('.__tmp','')
file_name_end = file_name_end
else:
file_name_end = file_name_end + ".__tmp"
new_file_name_end = file_name_end
new_sub_dir = os.path.join(str(datum.sequencing.technicalreplicate.appliedassay.assay.assay_type),
str(datum.sequencing.technicalreplicate.appliedassay.biosamplereplicate.biosample.biosample_id))
new_file_name = os.path.join(new_sub_dir, new_file_name_end)
new_file_path = os.path.join(settings.MEDIA_ROOT, new_file_name)
log_file.write("new file path: "+ new_file_path +"\n")
if not os.path.exists(os.path.join(settings.MEDIA_ROOT,new_sub_dir)):
os.makedirs(os.path.join(settings.MEDIA_ROOT,new_sub_dir))
os.rename(datum.primary_file.path, new_file_path)
log_file.write("finished moving file \n")
writer.writerow([datum.primary_file.path, new_file_path])
datum.primary_file.name = new_file_name
if hasattr(datum.secondary_file, 'path'):
log_file.write("old file path: "+ datum.secondary_file.path +"\n")
try:
file_name_end = datum.secondary_file.name.split('/')[2].split('.',2)[2]
if ".__tmp" in file_name_end:
# file_name_end = file_name_end.replace('.__tmp','')
file_name_end = file_name_end
else:
file_name_end = file_name_end + ".__tmp"
new_file_name_end = '.'.join(map(str, (datum.sequencing.technicalreplicate.appliedassay.assay.assay_id,
datum.sequencing.sequencing_id,file_name_end)))
except IndexError:
file_name_end = datum.secondary_file.name.split('/')[2]
if ".__tmp" in file_name_end:
# file_name_end = file_name_end.replace('.__tmp','')
file_name_end = file_name_end
else:
file_name_end = file_name_end + ".__tmp"
new_file_name_end = file_name_end
new_sub_dir = os.path.join(str(datum.sequencing.technicalreplicate.appliedassay.assay.assay_type),
str(datum.sequencing.technicalreplicate.appliedassay.biosamplereplicate.biosample.biosample_id))
new_file_name = os.path.join(new_sub_dir, new_file_name_end)
new_file_path = os.path.join(settings.MEDIA_ROOT, new_file_name)
log_file.write("new file path: "+ new_file_path +"\n")
if not os.path.exists(os.path.join(settings.MEDIA_ROOT,new_sub_dir)):
os.makedirs(os.path.join(settings.MEDIA_ROOT,new_sub_dir))
os.rename(datum.secondary_file.path, new_file_path)
log_file.write("finished moving file \n")
writer.writerow([datum.secondary_file.path, new_file_path])
datum.secondary_file.name = new_file_name
datum.save()
log_file.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") +": finished updating "+ datum.data_id + "\n")
log_file.write("------------------------------------------------------------" + "\n")
log_file.close()
# updateTemporaryFilePaths(request)
return response
[docs]@login_required
@user_passes_test(is_member)
def updateTemporaryFilePaths(request):
"""
Remove the ``__tmp`` suffix of the updated file names.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
json_data = request.body
json_data = json.loads(json_data)
if not json_data:
raise Http404("empty JSON")
this_data = []
log_file = open(os.path.join(settings.MEDIA_ROOT,'log_files','updatedFilePaths.log'),"a+")
success = True
for data_field in json_data:
try:
data = Data.objects.filter(sequencing__technicalreplicate__appliedassay__biosamplereplicate__biosample__series__series_id=data_field['series_id'])
except Data.DoesNotExist:
does_not_exist = True
return Http404("series does not exist")
for datum in data:
log_file.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") +": started updating "+ datum.data_id + "\n")
log_file.write("old file path: "+ datum.primary_file.path +"\n")
file_name_end = datum.primary_file.name.split('/')[2].split('.',2)[2]
file_name_end = file_name_end.replace('.__tmp','')
new_file_name_end = '.'.join(map(str, (datum.sequencing.technicalreplicate.appliedassay.assay.assay_id,
datum.sequencing.sequencing_id,file_name_end)))
new_sub_dir = os.path.join(str(datum.sequencing.technicalreplicate.appliedassay.assay.assay_type),
str(datum.sequencing.technicalreplicate.appliedassay.biosamplereplicate.biosample.biosample_id))
new_file_name = os.path.join(new_sub_dir, new_file_name_end)
new_file_path = os.path.join(settings.MEDIA_ROOT, new_file_name)
log_file.write("new file path: "+ new_file_path +"\n")
if not os.path.exists(os.path.join(settings.MEDIA_ROOT,new_sub_dir)):
os.makedirs(os.path.join(settings.MEDIA_ROOT,new_sub_dir))
os.rename(datum.primary_file.path, new_file_path)
log_file.write("finished moving file \n")
datum.primary_file.name = new_file_name
if hasattr(datum.secondary_file, 'path'):
log_file.write("old file path: "+ datum.secondary_file.path +"\n")
file_name_end = datum.secondary_file.name.split('/')[2].split('.',2)[2]
file_name_end = file_name_end.replace('.__tmp','')
new_file_name_end = '.'.join(map(str, (datum.sequencing.technicalreplicate.appliedassay.assay.assay_id,
datum.sequencing.sequencing_id,file_name_end)))
new_sub_dir = os.path.join(str(datum.sequencing.technicalreplicate.appliedassay.assay.assay_type),
str(datum.sequencing.technicalreplicate.appliedassay.biosamplereplicate.biosample.biosample_id))
new_file_name = os.path.join(new_sub_dir, new_file_name_end)
new_file_path = os.path.join(settings.MEDIA_ROOT, new_file_name)
log_file.write("new file path: "+ new_file_path +"\n")
if not os.path.exists(os.path.join(settings.MEDIA_ROOT,new_sub_dir)):
os.makedirs(os.path.join(settings.MEDIA_ROOT,new_sub_dir))
os.rename(datum.secondary_file.path, new_file_path)
log_file.write("finished moving file \n")
datum.secondary_file.name = new_file_name
datum.save()
log_file.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") +": finished updating "+ datum.data_id + "\n")
log_file.write("------------------------------------------------------------" + "\n")
log_file.close()
return JsonResponse(json.dumps({"success":success}), safe=False)
[docs]def updateAnalyzedData(request):
"""
Update analyzed data objects, if certain fields were missing or incorrect
during their initial upload.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
json_data = request.body
json_data = json.loads(json_data)
if not json_data:
raise Http404("empty JSON")
this_data = []
if not os.path.exists(os.path.join(settings.MEDIA_ROOT,'log_files')):
os.makedirs(os.path.join(settings.MEDIA_ROOT,'log_files'))
log_file = open(os.path.join(settings.MEDIA_ROOT,'log_files','DNANexus.log'),"a+")
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="updatedData.csv"'
writer = csv.writer(response)
duplicates = []
for data_field in json_data:
try:
data = Data.objects.filter(primary_source=data_field['primary_source'])
except Data.DoesNotExist:
does_not_exist = True
return Http404("data does not exist")
for datum in data:
log_file.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") +": started updating " + data_field['primary_source'].split("|")[0] + " with data_id "+ datum.data_id + "\n")
if datum.command =='':
datum.command = data_field['command']
if datum.mapped_genome_id is None:
datum.mapped_genome_id = 1
# if datum.version_id is None:
# datum.version = Versions.objects.get(name='freeze-01')
datum.save()
if data_field['derived_from_data'] != {}:
for derived_data in data_field['derived_from_data']:
datum.derived_from_data.add(
Data.objects.get(data_id=derived_data)
)
sequencing_id = datum.sequencing_id
sequencing_ref = Sequencing.objects.get(id=sequencing_id)
log_file.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") +": finished updating " + data_field['primary_source'].split("|")[0] + " with data_id "+ datum.data_id + "\n")
writer.writerow([datum.data_id,
datum.primary_file.name,
datum.primary_source,
datum.primary_file.size,
datum.command,
datum.version])
duplicates.append(datum.data_id)
for related_datum in sequencing_ref.sequencing_data.all():
if related_datum.data_id not in duplicates:
row = [related_datum.data_id,
related_datum.primary_file.name,
related_datum.primary_source,
related_datum.primary_file.size,
related_datum.command,
related_datum.version]
writer.writerow(row)
duplicates.append(related_datum.data_id)
log_file.close()
return response
[docs]@login_required
@user_passes_test(is_member)
def updateFile(request):
"""
Replace the current file attached to a data instance with a new file.
"""
if request.method != 'POST':
raise Http404("Must be a POST request!")
json_data = request.body
json_data = json.loads(json_data)
if not json_data:
raise Http404("empty JSON")
this_data = []
if not os.path.exists(os.path.join(settings.MEDIA_ROOT,'log_files')):
os.makedirs(os.path.join(settings.MEDIA_ROOT,'log_files'))
log_file = open(os.path.join(settings.MEDIA_ROOT,'log_files','DNANexus.log'),"a+")
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="updatedData.csv"'
writer = csv.writer(response)
for data_field in json_data:
try:
data = Data.objects.get(data_id=data_field['data_id'])
except Data.DoesNotExist:
does_not_exist = True
return Http404("data does not exist")
log_file.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") +": started updating " + data_field['primary_file'].split("|")[0] + " with data_id "+ data.data_id + "\n")
try:
primary_file, primary_source = get_file_object(
data_field['primary_file'], None)
except:
try:
primary_file, primary_source = get_file_object(
None, data_field['primary_file'])
except IOError:
return HttpResponseNotFound(data_field['primary_file'] +
" is not a valid URL or file path on the server")
data.primary_file=File(primary_file)
data.save()
log_file.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") +": finished updating " + data_field['primary_file'].split("|")[0] + " with data_id "+ data.data_id + "\n")
writer.writerow([data.data_id,
data.primary_file.name,
data.primary_file.size,
])
log_file.close()
return response
#####continue to batch upload#####
[docs]def get_fk_model(model, fieldname):
"""
Helper function to get the foreignKey for a given fieldname in a given model.
"""
field_object = model._meta.get_field(fieldname)
m2m = field_object.many_to_many
direct = not field_object or field_object.concrete
if not m2m and direct and isinstance(
field_object, ForeignKey):
return field_object.rel.to
return None
[docs]@login_required
@user_passes_test(is_member)
def batchUpload(request):
"""
Handle the batch upload by reading in the csv file, validating it and if all
tests pass, staging the upload for final submission.
"""
def validate_file(csv_file):
# check that a csv file was uploaded
try:
# use splitlines since FileField can't be opened in
# 'rU' mode
dialect = csv.Sniffer().sniff(
'\n'.join(csv_file.read(2048).encode(
'utf-8').splitlines()[:-1]))
csv_file.seek(0)
except csv.Error:
return 'Not a CSV file.', None, None
except UnicodeDecodeError as e:
return 'File contains non-unicode characters: ' + str(e), \
None, None
# read the csv into memory as utf-8
try:
reader = csv.reader(
csv_file.read().encode('utf-8').splitlines(), dialect)
# parse the header line
header = [f.replace('__', '00') for f in reader.next()]
# somehow a null byte can break the CSV file parser
except UnicodeDecodeError as e:
return 'File contains non-unicode characters: ' + str(e), \
None, None
# There was an error with a NUL byte, but hopefully the
# above except handles this error
# except Error as e:
# return "Error with CSV header: " + str(e), None, None
# validate header fields
if not all(f in VALID_BATCH_HEADERS for f in header):
invalid_headers_err = 'Invalid headers: ' + ', '.join(
f.replace("00", "__") for f in header
if f not in VALID_BATCH_HEADERS and f != "")
if any(f == "" for f in header):
if len(invalid_headers_err) > 17:
invalid_headers_err += ', [Empty header field]'
else:
invalid_headers_err += '[Empty header field]'
return invalid_headers_err, None, None
all_rows = list(reader)
return None, all_rows, header
def prepBatchUpload(all_rows, header):
def test_model_inst(
model_class, model_data, model_name, ref_fks, row_num):
model_errors = []
row_model = model_class()
for f, v in model_data.items():
# skip internal_id field
if f == 'internal_id':
continue
# handle controlled_by fields from appliedassay or
# biosample to add later
if f == 'controlled_by':
valid_data['control_' + model_name].append((
model_data['internal_id'], v, row_num))
continue
# handle foreign key fields
fk_model = get_fk_model(model_class, f)
field_type = model_class._meta.get_field(f)
if fk_model:
try:
setattr(row_model, f,
fk_model.objects.get(name=v))
except ObjectDoesNotExist:
model_errors.append(
'Controlled vocabulary field ' +
f + ' contains invalid value.')
elif isinstance(field_type, FloatField):
setattr(row_model, f, float(v))
elif isinstance(field_type, IntegerField):
setattr(row_model, f, int(v))
elif isinstance(field_type, BooleanField):
setattr(row_model, f, v in BATCH_FALSE_VALUES)
# not sure if other processing will be done in the
# future (year only, etc.)
elif isinstance(field_type, DateField):
setattr(row_model, f, v)
# if it is not a special field add as a charfiled
else:
setattr(row_model, f, v)
# save fk data separately since it won't have model name
# prepended as with model_data
fk_data = []
# set reference (biosamp -> series, etc.) forgein keys
for f, f_id in ref_fks:
# if this is the first data being added we need a
# special warning that the first data must be
# entered manually
dummie_fk_ref = get_fk_model(
model_class, f).objects.first()
if dummie_fk_ref is None:
model_errors.append(
'Batch upload cannot be the first data ' +
'entered. If data exists in the data base ' +
'please contact the administrators.')
else:
# add a dummie fk reference to this object in order
# to check the object (this will not be added
# to the DB)
setattr(row_model, f, dummie_fk_ref)
# if the reference is None the previus model instance
# is not valid, so don't add here
if f_id.name is not None:
# add the real fk_ref to the model_data
fk_data.append((f + "00" + f_id.name, f_id.value))
# set id, and user attributes so model will pass full_clean
setattr(row_model, model_name + '_id', 'ZCDfoo')
setattr(row_model, 'user', request.user)
# check that this instance is complete and valid
row_model_id = ID_FIELD(None, None)
try:
# controlled_by fields handled separately
if 'controlled_by' in model_data:
del model_data['controlled_by']
row_model.full_clean()
valid_data['internal_' + model_name][
model_data['internal_id']] = dict(
[(model_name + "00" + k, v)
for k, v in model_data.items()] + fk_data)
row_model_id = ID_FIELD(
'internal_id', model_data['internal_id'])
except ValidationError as e:
model_errors.append(
'Invalid ' + model_name + ' model instance (most ' +
'likely missing required fields or ' +
'mis-formated data type). Possible ' +
'fields: ' + ', '.join(map(str, set([
k for k, v in e]))) + '.')
return model_errors, row_model_id
def process_model_fields(
model_data, model_class, ref_fks=[], row_num=None):
model_name = model_class.__name__.lower()
row_model_id, row_errors = ID_FIELD(None, None), []
if model_name + '_id' in model_data.keys():
# if id is provided no other fields from that model
# should be passed
if len(model_data) > 1:
row_errors.append(
'Can not provide ' +
'{0}_id and other {0} fields.'.format(
model_name))
# try to locate this entry in the db
try:
row_model = model_class.objects.get(
**{model_name + '_id':
model_data[model_name + '_id']})
row_model_id = ID_FIELD(
model_name + '_id', model_data[
model_name + '_id'])
# check the the user updating this is the owner
if row_model.user != request.user:
row_errors.append(
'You are not the owner of the ' +
model_name + ' annotation object ' +
'referenced in this row.')
except ObjectDoesNotExist:
row_errors.append(
model_name + ' ID does not exist.')
# if this is a new model
else:
if 'internal_id' not in model_data.keys():
row_errors.append(
'Must provide either a ' +
'{0}__{0}_id or {0}__internal_id'.format(
model_name) + ' if any ' + model_name +
' fields are supplied.')
# model already exists in dict
elif model_data['internal_id'] in \
valid_data['internal_' + model_name]:
row_model_id = ID_FIELD(
'internal_id', model_data['internal_id'])
# validate that any fields entered match
# previous entry
prev_model = valid_data[
'internal_' + model_name][row_model_id.value]
if not (set((
model_name + "00" + f
for f in model_data.keys())).issubset(
prev_model.keys()) and
all(v == prev_model[
model_name + "00" + f]
for f, v in model_data.items()
if v)):
row_errors.append(
model_name + ' data does not match ' +
'previously entered data for this id.')
# else this is a new entry to add to the database
else:
model_errors, model_id = test_model_inst(
model_class, model_data, model_name,
ref_fks, row_num)
row_errors.extend(model_errors)
row_model_id = model_id
return row_model_id, row_errors
def validate_url(url):
# request the HEAD and check the code and status/msg
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
try:
# not currently supporting FTP but can add access here
# if requested
if not url.startswith('http://') and \
not url.startswith('https://'):
url = "https://" + url
res = urllib2.urlopen(HeadRequest(url), timeout=1)
if res.code != 200 or res.msg != "OK":
return False
except urllib2.URLError:
return False
except socket.timeout:
return False
return True
def validate_path(local_path):
if local_path.startswith('/'):
local_path = local_path[1:]
if os.path.isfile(os.path.join(UPLOAD_PATH, local_path)):
return True
return False
def process_data(data_data, paired_end, seq_id):
data_errors = []
if set(data_data).isdisjoint(('url', 'local_path')):
data_errors.append(
'Must provide either a URL or local path for ' +
'genomic data files.')
elif 'url' in data_data and not validate_url(
data_data['url']):
data_errors.append(
'File URL does not exist or is not accessible ' +
'(Note FTP is not currently supported).')
elif 'local_path' in data_data and not validate_path(
data_data['local_path']):
data_errors.append(
'File path does not exist on the DCC ' +
'server.')
# check for secondary files if this is paired end
if paired_end:
if set(data_data).isdisjoint((
'secondary_url', 'secondary_local_path')):
data_errors.append(
'Must provide either a secondary URL or ' +
'local path for paired-end genomic data files.')
elif 'secondary_url' in data_data and not validate_url(
data_data['secondary_url']):
data_errors.append(
'Secondary URL does not exist or is not ' +
'accessible (Note FTP is not currently ' +
'supported).')
elif 'local_path' in data_data and not validate_path(
data_data['local_path']):
data_errors.append(
'File path does not exist on the DCC' +
' server.')
if len(data_errors) == 0 and seq_id.name is not None:
valid_data['data'][len(valid_data['data'])] = dict(
[("data00" + k, v) for k, v in data_data.items()] +
[("sequencing00" + seq_id.name, seq_id.value), ])
return data_errors
def check_cond_reqs(biosamp_data, aa_data, seq_data):
cond_req_errors = []
if 'biosample_type' in biosamp_data and \
biosamp_data['biosample_type'] == 'tissue' and \
'anatomical_term' not in biosamp_data:
cond_req_errors.append(
'Invalid biosample model instance. If biosample_' +
'type is tissue then a anatomical_term must ' +
'be provided')
if 'biosample_type' in biosamp_data and biosamp_data[
'biosample_type'] in ('tissue', 'whole organism'):
if 'stage' not in biosamp_data:
cond_req_errors.append(
'Invalid biosample model instance. If ' +
'biosample_type is tissue or whole organism ' +
'then a stage must be provided')
if 'post_fertilization' not in biosamp_data:
cond_req_errors.append(
'Invalid biosample model instance. If ' +
'biosample_type is tissue or whole organism ' +
'then a post_fertilization must be provided')
if 'biosample_type' in biosamp_data and \
biosamp_data['biosample_type'] == 'cell line' and \
'cell_line_type' not in biosamp_data:
cond_req_errors.append(
'Invalid biosample model instance. If biosample_' +
'type is cell line then a cell_line_type must ' +
'be provided')
if 'assay_type' in assay_data and assay_data['assay_type'] in \
TARGET_REQD_ASSAYS and 'target' not in assay_data:
cond_req_errors.append(
'Invalid assay model instance. If assay_type ' +
'requires a target, e.g., ChIP-seq,' +
' then a target must be provided')
if 'assay_type' in assay_data and assay_data['assay_type'] in \
RNA_BASED_ASSAYS and 'library_prep' not in assay_data:
cond_req_errors.append(
'Invalid assay model instance. If assay_type ' +
'requires a library_prep, e.g., RNA-seq,'+
' then a library_prep method must be provided')
if 'assay_type' in assay_data and assay_data['assay_type'] \
in RNA_BASED_ASSAYS and 'strand_mode' not in seq_data:
cond_req_errors.append(
'Invalid sequencing model instance. If assay ' +
'type is RNA-based then a strand_mode must ' +
'be provided')
return cond_req_errors
def verify_controls():
for internal_id, controlled_by, row_num in \
valid_data['control_biosample']:
if controlled_by not in \
valid_data['internal_biosample'].keys() + \
[biosamp.biosample_id for biosamp in
Biosample.objects.all()]:
all_row_errors[row_num].append(
'Controlled by value does not match any ' +
'internal or biosample ids.')
for internal_id, controlled_by, row_num in \
valid_data['control_appliedassay']:
if controlled_by not in \
valid_data['internal_appliedassay'].keys() + \
[aa.appliedassay_id for aa in
AppliedAssay.objects.all()]:
all_row_errors[row_num].append(
'Controlled by value does not match any ' +
'internal or applied assay ids.')
# fix valid data for upload to batch details
valid_data['control_biosample'] = dict((i, {
'biosample00internal_id':
valid_data['control_biosample'][i][0],
'biosample00controlled_by':
valid_data['control_biosample'][i][1]}
) for i in range(len(valid_data['control_biosample'])))
valid_data['control_appliedassay'] = dict((i, {
'appliedassay00internal_id':
valid_data['control_appliedassay'][i][0],
'appliedassay00controlled_by':
valid_data['control_appliedassay'][i][1]}
) for i in range(len(valid_data['control_appliedassay'])))
return
all_row_errors = []
valid_data = {
'internal_series': {},
'internal_biosample': {},
'control_biosample': [],
'internal_biosamplereplicate': {},
'internal_assay': {},
'internal_appliedassay': {},
'control_appliedassay': [],
'internal_technicalreplicate': {},
'internal_sequencing': {},
'data': {}}
# record entered biosamples and applied assays
# to ensure they are globablly unique identifiers
biosamp_ids = {}
aa_ids = {}
id_errors = []
for row_num, entry in enumerate(all_rows):
if len(entry) > len(header):
all_row_errors.append(['More fields than header.'])
continue
# create dictionary with filled values in this row
fields = dict((f_name, val) for f_name, val in
zip(header, entry) if val)
series_data, biosamp_data, biorep_data, assay_data, \
aa_data, techrep_data, seq_data, data_data \
= [dict((f.replace(model_name + "00", ""), v)
for f, v in fields.items()
if re.match(model_name + "00", f))
for model_name in (
'series', 'biosample', 'biosamplereplicate',
'assay', 'appliedassay',
'technicalreplicate', 'sequencing', 'data')]
if sum(map(len, (series_data, biosamp_data, biorep_data,
assay_data, aa_data,
techrep_data, data_data))) == 0:
all_row_errors.append([
'Row must include annotation for series, ' +
'biosample, assay, sequencing and data.'])
continue
series_id, biosamp_id, biorep_id, assay_id, aa_id, \
techrep_id, seq_id, data_id \
= [ID_FIELD(None, None), ] * 8
series_id, series_errors = process_model_fields(
series_data, Series)
if 'internal_id' in biosamp_data:
if biosamp_data['internal_id'] in biosamp_ids:
if series_id.value != \
biosamp_ids[biosamp_data['internal_id']]:
id_errors.append(
'Cannot add the same biosample ' +
'(internal_id) to different series :' +
str(series_id.value) + ' != ' +
str(biosamp_ids[
biosamp_data['internal_id']]))
else:
biosamp_ids[biosamp_data['internal_id']] \
= series_id.value
biosamp_id, biosamp_errors = process_model_fields(
biosamp_data, Biosample, (('series', series_id),),
row_num)
# add biosample to bio_rep id as they are nested
if 'internal_id' in biorep_data:
biorep_data['internal_id'] \
= 'BS' + str(biosamp_id.value) + '_' + \
biorep_data['internal_id']
biorep_id, biorep_errors = process_model_fields(
biorep_data, BiosampleReplicate,
(('biosample', biosamp_id),))
assay_id, assay_errors = process_model_fields(
assay_data, Assay)
if 'internal_id' in aa_data:
if aa_data['internal_id'] in aa_ids:
if (biorep_id.value, assay_id.value) != \
aa_ids[aa_data['internal_id']]:
id_errors.append(
'Cannot add the same applied assay ' +
'(internal_id) with different linked ' +
'biosample replicates or assays: ' + str(
(biorep_id.value, assay_id.value)) +
' != ' + str(aa_ids[
aa_data['internal_id']]))
else:
aa_ids[aa_data['internal_id']] \
= (biorep_id.value, assay_id.value)
aa_id, aa_errors = process_model_fields(
aa_data, AppliedAssay,
(('assay', assay_id),
('biosamplereplicate', biorep_id)), row_num)
if 'internal_id' in techrep_data:
techrep_data['internal_id'] \
= 'AA' + str(aa_id.value) + '_' + \
techrep_data['internal_id']
techrep_id, techrep_errors = process_model_fields(
techrep_data, TechnicalReplicate,
(('appliedassay', aa_id),))
if 'internal_id' in seq_data:
seq_data['internal_id'] \
= 'TR' + str(techrep_id.value) + '_' + \
seq_data['internal_id']
seq_id, seq_errors = process_model_fields(
seq_data, Sequencing,
(('technicalreplicate', techrep_id),))
# check conditional required fields
cond_req_errors = check_cond_reqs(
biosamp_data, aa_data, seq_data)
# process genomic files
paired_end = seq_data['paired_end'] not in \
BATCH_FALSE_VALUES if 'paired_end' \
in seq_data else False
data_errors = process_data(data_data, paired_end, seq_id)
# add row_errors that were accumulated on this row
all_row_errors.append(
series_errors + biosamp_errors + biorep_errors +
assay_errors + aa_errors + techrep_errors + seq_errors +
data_errors + cond_req_errors + id_errors)
# process the controlled by biosamples and applied_assays now
# that all new model objects are recorded
verify_controls()
# if there are any errors at all return those
# errors for correction
if sum([len(row_errors) for row_errors in all_row_errors]) > 0:
return False, {'rows': all_rows, 'errors': all_row_errors}
# else return the correctly parsed data
return True, valid_data
def load_to_insert(valid_data, csvForm):
# update the batch upload object and save
batch_inst = csvForm.save(commit=False)
batch_inst.user = request.user
batch_inst.isValid = True
batch_inst.committed = False
batch_inst.save()
# add all objects to the batch details table
for model_key, model_name in zip(
('internal_series', 'internal_biosample',
'internal_biosamplereplicate', 'internal_assay',
'internal_appliedassay', 'internal_technicalreplicate',
'internal_sequencing', 'data', 'control_biosample',
'control_appliedassay'),
('series', 'biosample', 'biosamplereplicate',
'assay', 'appliedassay', 'technicalreplicate',
'sequencing', 'data', 'bioControl', 'aaControl')):
for model_data in valid_data[model_key].values():
row_model = BatchUploadDetails(**model_data)
row_model.batchUpload = batch_inst
row_model.row_type = model_name
row_model.save()
batch_q = BatchUploadDetails.objects.filter(
batchUpload__pk=batch_inst.pk)
batch_fields = set([
f.name for f in batch_q[0]._meta.get_fields()]).difference(
('id', 'batchUpload', 'errors', 'row_type'))
def get_rep_to_seq_qs(bs_id):
aa_to_seq_data = []
for br_q, aa_q, tr_q, sq_q, dt_q in [
(br_q, aa_q, tr_q, sq_q, dt_q)
for br_q in batch_q.filter(
row_type='biosamplereplicate').filter(
biosample00internal_id=bs_id)
for aa_q in batch_q.filter(
row_type='appliedassay').filter(
biosamplereplicate00internal_id=br_q.biosamplereplicate00internal_id)
for tr_q in batch_q.filter(
row_type='technicalreplicate').filter(
appliedassay00internal_id=aa_q.appliedassay00internal_id)
for sq_q in batch_q.filter(
row_type='sequencing').filter(
technicalreplicate00internal_id=tr_q.technicalreplicate00internal_id)
for dt_q in batch_q.filter(
row_type='data').filter(
sequencing00internal_id=sq_q.sequencing00internal_id)]:
dataTab = BatchUploadTable(
batch_q.filter(id=dt_q.id), orderable=False)
dataTab.exclude = set(
('id', 'row_type', 'errors',
'sequencing00internal_id')).union(
f for f in batch_fields if getattr(
batch_q.get(id=dt_q.id), f) is None)
# copy sequencing object
all_data_q = sq_q
all_data_q.pk = None
# then add in elements from other objects
for q_obj in (tr_q, aa_q, br_q):
for f in batch_fields:
if getattr(q_obj, f):
setattr(
all_data_q, f, getattr(q_obj, f))
# save psuedo row in order to show, but remove
# row_type so it isn't added to real DB
all_data_q.row_type = None
all_data_q.save()
aa2seqTab = BatchUploadTable(
batch_q.filter(id=all_data_q.id), orderable=False)
aa2seqTab.exclude = set(
('id', 'row_type', 'errors',
'biosample00internal_id')).union(
f for f in batch_fields if getattr(
all_data_q, f) is None)
aa_to_seq_data.append((aa2seqTab, dataTab))
return aa_to_seq_data
series_nested_tables = []
for series_q in BatchUploadDetails.objects.filter(
batchUpload__pk=batch_inst.pk).filter(
row_type='series'):
seriesTab = BatchUploadTable(
batch_q.filter(id=series_q.id), orderable=False)
seriesTab.exclude = set(
('id', 'row_type', 'errors')).union(
f for f in batch_fields if getattr(
series_q, f) is None)
series_data = []
for bs_q in BatchUploadDetails.objects.filter(
batchUpload__pk=batch_inst.pk).filter(
row_type='biosample').filter(
series00internal_id=series_q.series00internal_id):
biosampTab = BatchUploadTable(
batch_q.filter(id=bs_q.id), orderable=False)
biosampTab.exclude = set(
('id', 'row_type', 'errors',
'series00internal_id')).union(
f for f in batch_fields if getattr(
bs_q, f) is None)
# get all other annotation data and nested data
# objects under this biosample
series_data.append((biosampTab, get_rep_to_seq_qs(
bs_q.biosample00internal_id)))
series_nested_tables.append((seriesTab, series_data))
assayTab = BatchUploadTable(
batch_q.filter(row_type='assay'), orderable=False)
assayTab.exclude = set(
('id', 'row_type', 'errors')).union(
f for f in batch_fields if all(
getattr(as_q, f) is None
for as_q in batch_q.filter(row_type='assay')))
all_tables = {
'series': series_nested_tables,
'assayTab': assayTab,
'batchId': batch_inst.pk}
return all_tables
def load_to_correct(data_and_errors, header, csvForm):
# update the batch upload object and save
batch_inst = csvForm.save(commit=False)
batch_inst.user = request.user
batch_inst.isValid = False
batch_inst.committed = False
batch_inst.save()
all_rows = data_and_errors['rows']
all_row_errors = data_and_errors['errors']
for row_data, row_errors in zip(all_rows, all_row_errors):
row_model = BatchUploadDetails(**dict(
zip(header, map(lambda x: x[:100], row_data))))
row_model.batchUpload = batch_inst
row_model.errors = '<br>'.join(
'<span class="fas fa-exclamation-triangle" ' +
'aria-hidden="true"></span> Error ' +
str(i + 1) + ": " + error for i, error in
enumerate(row_errors[:10]))
row_model.save()
batchTab = BatchUploadTable(BatchUploadDetails.objects.filter(
batchUpload__pk=batch_inst.pk), order_by="id")
batchTab.exclude = set(
f.name for f in
BatchUploadDetails._meta.get_fields()).difference(
header + ['errors'])
return batchTab
if request.method == 'POST':
csvForm = BatchUploadForm(request.POST, request.FILES)
if csvForm.is_valid():
file_error, all_rows, header = validate_file(
request.FILES['csvFile'])
if file_error:
return render(
request, 'modeldcc/batchUploadFileError.html',
{'errorMessage': file_error,
'batchForm': BatchUploadForm()})
batch_valid, parsed_rows = prepBatchUpload(all_rows, header)
# if the batch upload was processed successfully then
# process the rows in preparation for upload
if batch_valid:
all_tables = load_to_insert(parsed_rows, csvForm)
return render(
request, 'modeldcc/batchUploadSuccess.html',
all_tables)
# if the batch upload was unsuccessful then parse the
# rows with the errors to be corrected
else:
batchTab = load_to_correct(parsed_rows, header, csvForm)
return render(
request, 'modeldcc/batchUploadFail.html',
{'batchTab': batchTab,
'batchForm': BatchUploadForm()})
else:
return render(
request, 'modeldcc/batchUploadFileError.html',
{'errorMessage': 'Must enter a CSV file.',
'batchForm': BatchUploadForm()})
# if a get request then return the CSV upload form
return render(request, 'modeldcc/batchUpload.html',
{'batchForm': BatchUploadForm()})
[docs]@login_required
@user_passes_test(is_member)
def commitBatch(request, batch_pk):
"""
If all checks passed and the user also checked parsing of the csv file,
add the batch staged annotations including the sequencing files to the DCC
"""
def convert_field(model_class, field, value):
# handle foreign key fields
fk_model = get_fk_model(model_class, field)
if fk_model:
return fk_model.objects.get(name=value)
elif isinstance(model_class._meta.get_field(field),
FloatField):
return float(value)
elif isinstance(model_class._meta.get_field(field),
IntegerField):
return int(value)
elif isinstance(model_class._meta.get_field(field),
BooleanField):
return value not in BATCH_FALSE_VALUES
# if it is not a special field add as a charfiled
return value
def add_anno_element(model_class, fk_refs=[]):
model_name = model_class.__name__.lower()
for ref_data in batch_elements.filter(row_type=model_name):
# get the fields for this model that have values
load_fields = [(f, f.replace(model_name + "00", ""))
for f in all_fields
if f.startswith(model_name) and
getattr(ref_data, f) and
f != model_name + "00internal_id"]
data_to_load = dict(
(model_f, convert_field(model_class, model_f,
getattr(ref_data, f)))
for f, model_f in load_fields)
for fk_model in fk_refs:
fk_mn = fk_model.__name__.lower()
fk_ref_n = fk_mn + "00" + fk_mn + "_id"
fk_int_n = fk_mn + "00internal_id"
# get either previously annotated fk ref or fk ref
# from object just created with internal_id
fk_value = fk_model.objects.get(**{
fk_mn + "_id": getattr(
ref_data, fk_ref_n)}) \
if getattr(ref_data, fk_ref_n) else \
fk_model.objects.get(**{
fk_mn + "_id": id_conv[fk_mn][getattr(
ref_data, fk_int_n)]})
data_to_load[fk_mn] = fk_value
data_to_load['user'] = request.user
model_inst = model_class(**data_to_load)
model_inst.save()
id_conv[model_name][getattr(
ref_data, model_name + "00internal_id")] \
= getattr(model_inst, model_name + "_id")
return
try:
batch_upload_obj = BatchUploads.objects.get(pk=batch_pk)
except ObjectDoesNotExist:
return render(
request, 'modeldcc/batchUploadFileError.html',
{'errorMessage': 'This batch id does not exist. ' +
'To upload a new batch annotation please enter a ' +
'valid CSV below.',
'batchForm': BatchUploadForm()})
# don't let the same upload be commited twice
if batch_upload_obj.committed:
return render(
request, 'modeldcc/batchUploadFileError.html',
{'errorMessage': 'This batch has already been committed. ' +
'To upload a new batch annotation please enter a ' +
'valid CSV below.',
'batchForm': BatchUploadForm()})
if not batch_upload_obj.isValid:
return render(
request, 'modeldcc/batchUploadFileError.html',
{'errorMessage': 'This batch was not a valid CSV. ' +
'To upload a new batch annotation please enter a ' +
'valid CSV below.',
'batchForm': BatchUploadForm()})
batch_elements = BatchUploadDetails.objects.filter(
batchUpload=batch_upload_obj)
id_conv = {'series': {}, 'biosample': {}, 'biosamplereplicate': {},
'assay': {}, 'appliedassay': {}, 'technicalreplicate': {},
'sequencing': {}}
all_fields = [f.name for f in BatchUploadDetails._meta.get_fields()]
add_anno_element(Series)
# get all series entries being added here
added_series = [id_conv['series'][int_series_id]
for int_series_id in
batch_elements.values_list(
'series00internal_id', flat=True).distinct()
if int_series_id is not None]
add_anno_element(Biosample, [Series, ])
add_anno_element(BiosampleReplicate, [Biosample, ])
add_anno_element(Assay)
add_anno_element(AppliedAssay, [BiosampleReplicate, Assay])
add_anno_element(TechnicalReplicate, [AppliedAssay, ])
add_anno_element(Sequencing, [TechnicalReplicate, ])
for data_data in batch_elements.filter(row_type='data'):
seq_id = data_data.sequencing00sequencing_id \
if data_data.sequencing00sequencing_id else \
id_conv['sequencing'][
data_data.sequencing00internal_id]
seq_value = Sequencing.objects.get(sequencing_id=seq_id)
primary_file, primary_source = get_file_object(
data_data.data00url, data_data.data00local_path)
secondary_file, secondary_source = get_file_object(
data_data.data00secondary_url,
data_data.data00secondary_local_path)
file_inst = Data(
primary_file=File(primary_file),
secondary_file=File(secondary_file),
primary_source=primary_source,
secondary_source=secondary_source,
sequencing=seq_value,
user=request.user)
file_inst.save()
primary_file.close()
if secondary_file is not None:
secondary_file.close()
# update Biosamples and AppliedAssays with controlled_by values
for bio_cont_data in batch_elements.filter(row_type='bioControl'):
cont_by_str = bio_cont_data.biosample00controlled_by
cont_by_id = cont_by_str if cont_by_str not in id_conv[
'biosample'] else id_conv['biosample'][cont_by_str]
cont_by_value = Biosample.objects.get(biosample_id=cont_by_id)
to_update_value = Biosample.objects.get(
biosample_id=id_conv['biosample'][
bio_cont_data.biosample00internal_id])
to_update_value.controlled_by = cont_by_value
to_update_value.save()
for aa_cont_data in batch_elements.filter(row_type='aaControl'):
cont_by_str = aa_cont_data.appliedassay00controlled_by
cont_by_id = cont_by_str if cont_by_str not in id_conv[
'appliedassay'] else id_conv['appliedassay'][cont_by_str]
cont_by_value = AppliedAssay.objects.get(
appliedassay_id=cont_by_id)
to_update_value = AppliedAssay.objects.get(
appliedassay_id=id_conv['appliedassay'][
aa_cont_data.appliedassay00internal_id])
to_update_value.controlled_by = cont_by_value
to_update_value.save()
series_ids = []
for series_id in added_series:
series_ref = Series.objects.get(series_id=series_id)
series_ref.status = statuses.objects.get(
name=COMPLETED_UPLOAD_STATUS)
series_ref.save()
series_ids.append([series_id, series_ref.pk, series_ref.title])
batch_upload_obj.committed = True
batch_upload_obj.save()
createDataExportJSON(request)
createindexJSON(request)
return render(
request, 'modeldcc/batchUploadCommit.html',
{'added_series':series_ids})