# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Multi-threaded word2vec mini-batched skip-gram model. Trains the model described in: (Mikolov, et. al.) Efficient Estimation of Word Representations in Vector Space ICLR 2013. http://arxiv.org/abs/1301.3781 This model does traditional minibatching. The key ops used are: * placeholder for feeding in tensors for each example. * embedding_lookup for fetching rows from the embedding matrix. * sigmoid_cross_entropy_with_logits to calculate the loss. * GradientDescentOptimizer for optimizing the loss. * skipgram custom op that does input processing. """ from __future__ import absolute_import from __future__ import division from __future__ import print_function import os import sys import threading import time from six.moves import xrange # pylint: disable=redefined-builtin import numpy as np import tensorflow as tf from tensorflow.models.embedding import gen_word2vec as word2vec flags = tf.app.flags flags.DEFINE_string("save_path", None, "Directory to write the model and " "training summaries.") flags.DEFINE_string("train_data", None, "Training text file. " "E.g., unzipped file http://mattmahoney.net/dc/text8.zip.") flags.DEFINE_string( "eval_data", None, "File consisting of analogies of four tokens." "embedding 2 - embedding 1 + embedding 3 should be close " "to embedding 4." "E.g. https://word2vec.googlecode.com/svn/trunk/questions-words.txt.") flags.DEFINE_integer("embedding_size", 200, "The embedding dimension size.") flags.DEFINE_integer( "epochs_to_train", 15, "Number of epochs to train. Each epoch processes the training data once " "completely.") flags.DEFINE_float("learning_rate", 0.2, "Initial learning rate.") flags.DEFINE_integer("num_neg_samples", 100, "Negative samples per training example.") flags.DEFINE_integer("batch_size", 16, "Number of training examples processed per step " "(size of a minibatch).") flags.DEFINE_integer("concurrent_steps", 12, "The number of concurrent training steps.") flags.DEFINE_integer("window_size", 5, "The number of words to predict to the left and right " "of the target word.") flags.DEFINE_integer("min_count", 5, "The minimum number of word occurrences for it to be " "included in the vocabulary.") flags.DEFINE_float("subsample", 1e-3, "Subsample threshold for word occurrence. Words that appear " "with higher frequency will be randomly down-sampled. Set " "to 0 to disable.") flags.DEFINE_boolean( "interactive", False, "If true, enters an IPython interactive session to play with the trained " "model. E.g., try model.analogy('france', 'paris', 'russia') and " "model.nearby(['proton', 'elephant', 'maxwell']") flags.DEFINE_integer("statistics_interval", 5, "Print statistics every n seconds.") flags.DEFINE_integer("summary_interval", 5, "Save training summary to file every n seconds (rounded " "up to statistics interval.") flags.DEFINE_integer("checkpoint_interval", 600, "Checkpoint the model (i.e. save the parameters) every n " "seconds (rounded up to statistics interval.") FLAGS = flags.FLAGS class Options(object): """Options used by our word2vec model.""" def __init__(self): # Model options. # Embedding dimension. self.emb_dim = FLAGS.embedding_size # Training options. # The training text file. self.train_data = FLAGS.train_data # Number of negative samples per example. self.num_samples = FLAGS.num_neg_samples # The initial learning rate. self.learning_rate = FLAGS.learning_rate # Number of epochs to train. After these many epochs, the learning # rate decays linearly to zero and the training stops. self.epochs_to_train = FLAGS.epochs_to_train # Concurrent training steps. self.concurrent_steps = FLAGS.concurrent_steps # Number of examples for one training step. self.batch_size = FLAGS.batch_size # The number of words to predict to the left and right of the target word. self.window_size = FLAGS.window_size # The minimum number of word occurrences for it to be included in the # vocabulary. self.min_count = FLAGS.min_count # Subsampling threshold for word occurrence. self.subsample = FLAGS.subsample # How often to print statistics. self.statistics_interval = FLAGS.statistics_interval # How often to write to the summary file (rounds up to the nearest # statistics_interval). self.summary_interval = FLAGS.summary_interval # How often to write checkpoints (rounds up to the nearest statistics # interval). self.checkpoint_interval = FLAGS.checkpoint_interval # Where to write out summaries. self.save_path = FLAGS.save_path # Eval options. # The text file for eval. self.eval_data = FLAGS.eval_data class Word2Vec(object): """Word2Vec model (Skipgram).""" def __init__(self, options, session): self._options = options self._session = session self._word2id = {} self._id2word = [] self.build_graph() self.build_eval_graph() self.save_vocab() self._read_analogies() def _read_analogies(self): """Reads through the analogy question file. Returns: questions: a [n, 4] numpy array containing the analogy question's word ids. questions_skipped: questions skipped due to unknown words. """ questions = [] questions_skipped = 0 with open(self._options.eval_data, "rb") as analogy_f: for line in analogy_f: if line.startswith(b":"): # Skip comments. continue words = line.strip().lower().split(b" ") ids = [self._word2id.get(w.strip()) for w in words] if None in ids or len(ids) != 4: questions_skipped += 1 else: questions.append(np.array(ids)) print("Eval analogy file: ", self._options.eval_data) print("Questions: ", len(questions)) print("Skipped: ", questions_skipped) self._analogy_questions = np.array(questions, dtype=np.int32) def forward(self, examples, labels): """Build the graph for the forward pass.""" opts = self._options # Declare all variables we need. # Embedding: [vocab_size, emb_dim] init_width = 0.5 / opts.emb_dim emb = tf.Variable( tf.random_uniform( [opts.vocab_size, opts.emb_dim], -init_width, init_width), name="emb") self._emb = emb # Softmax weight: [vocab_size, emb_dim]. Transposed. sm_w_t = tf.Variable( tf.zeros([opts.vocab_size, opts.emb_dim]), name="sm_w_t") # Softmax bias: [emb_dim]. sm_b = tf.Variable(tf.zeros([opts.vocab_size]), name="sm_b") # Global step: scalar, i.e., shape []. self.global_step = tf.Variable(0, name="global_step") # Nodes to compute the nce loss w/ candidate sampling. labels_matrix = tf.reshape( tf.cast(labels, dtype=tf.int64), [opts.batch_size, 1]) # Negative sampling. sampled_ids, _, _ = (tf.nn.fixed_unigram_candidate_sampler( true_classes=labels_matrix, num_true=1, num_sampled=opts.num_samples, unique=True, range_max=opts.vocab_size, distortion=0.75, unigrams=opts.vocab_counts.tolist())) # Embeddings for examples: [batch_size, emb_dim] example_emb = tf.nn.embedding_lookup(emb, examples) # Weights for labels: [batch_size, emb_dim] true_w = tf.nn.embedding_lookup(sm_w_t, labels) # Biases for labels: [batch_size, 1] true_b = tf.nn.embedding_lookup(sm_b, labels) # Weights for sampled ids: [num_sampled, emb_dim] sampled_w = tf.nn.embedding_lookup(sm_w_t, sampled_ids) # Biases for sampled ids: [num_sampled, 1] sampled_b = tf.nn.embedding_lookup(sm_b, sampled_ids) # True logits: [batch_size, 1] true_logits = tf.reduce_sum(tf.mul(example_emb, true_w), 1) + true_b # Sampled logits: [batch_size, num_sampled] # We replicate sampled noise lables for all examples in the batch # using the matmul. sampled_b_vec = tf.reshape(sampled_b, [opts.num_samples]) sampled_logits = tf.matmul(example_emb, sampled_w, transpose_b=True) + sampled_b_vec return true_logits, sampled_logits def nce_loss(self, true_logits, sampled_logits): """Build the graph for the NCE loss.""" # cross-entropy(logits, labels) opts = self._options true_xent = tf.nn.sigmoid_cross_entropy_with_logits( true_logits, tf.ones_like(true_logits)) sampled_xent = tf.nn.sigmoid_cross_entropy_with_logits( sampled_logits, tf.zeros_like(sampled_logits)) # NCE-loss is the sum of the true and noise (sampled words) # contributions, averaged over the batch. nce_loss_tensor = (tf.reduce_sum(true_xent) + tf.reduce_sum(sampled_xent)) / opts.batch_size return nce_loss_tensor def optimize(self, loss): """Build the graph to optimize the loss function.""" # Optimizer nodes. # Linear learning rate decay. opts = self._options words_to_train = float(opts.words_per_epoch * opts.epochs_to_train) lr = opts.learning_rate * tf.maximum( 0.0001, 1.0 - tf.cast(self._words, tf.float32) / words_to_train) self._lr = lr optimizer = tf.train.GradientDescentOptimizer(lr) train = optimizer.minimize(loss, global_step=self.global_step, gate_gradients=optimizer.GATE_NONE) self._train = train def build_eval_graph(self): """Build the eval graph.""" # Eval graph # Each analogy task is to predict the 4th word (d) given three # words: a, b, c. E.g., a=italy, b=rome, c=france, we should # predict d=paris. # The eval feeds three vectors of word ids for a, b, c, each of # which is of size N, where N is the number of analogies we want to # evaluate in one batch. analogy_a = tf.placeholder(dtype=tf.int32) # [N] analogy_b = tf.placeholder(dtype=tf.int32) # [N] analogy_c = tf.placeholder(dtype=tf.int32) # [N] # Normalized word embeddings of shape [vocab_size, emb_dim]. nemb = tf.nn.l2_normalize(self._emb, 1) # Each row of a_emb, b_emb, c_emb is a word's embedding vector. # They all have the shape [N, emb_dim] a_emb = tf.gather(nemb, analogy_a) # a's embs b_emb = tf.gather(nemb, analogy_b) # b's embs c_emb = tf.gather(nemb, analogy_c) # c's embs # We expect that d's embedding vectors on the unit hyper-sphere is # near: c_emb + (b_emb - a_emb), which has the shape [N, emb_dim]. target = c_emb + (b_emb - a_emb) # Compute cosine distance between each pair of target and vocab. # dist has shape [N, vocab_size]. dist = tf.matmul(target, nemb, transpose_b=True) # For each question (row in dist), find the top 4 words. _, pred_idx = tf.nn.top_k(dist, 4) # Nodes for computing neighbors for a given word according to # their cosine distance. nearby_word = tf.placeholder(dtype=tf.int32) # word id nearby_emb = tf.gather(nemb, nearby_word) nearby_dist = tf.matmul(nearby_emb, nemb, transpose_b=True) nearby_val, nearby_idx = tf.nn.top_k(nearby_dist, min(1000, self._options.vocab_size)) # Nodes in the construct graph which are used by training and # evaluation to run/feed/fetch. self._analogy_a = analogy_a self._analogy_b = analogy_b self._analogy_c = analogy_c self._analogy_pred_idx = pred_idx self._nearby_word = nearby_word self._nearby_val = nearby_val self._nearby_idx = nearby_idx def build_graph(self): """Build the graph for the full model.""" opts = self._options # The training data. A text file. (words, counts, words_per_epoch, self._epoch, self._words, examples, labels) = word2vec.skipgram(filename=opts.train_data, batch_size=opts.batch_size, window_size=opts.window_size, min_count=opts.min_count, subsample=opts.subsample) (opts.vocab_words, opts.vocab_counts, opts.words_per_epoch) = self._session.run([words, counts, words_per_epoch]) opts.vocab_size = len(opts.vocab_words) print("Data file: ", opts.train_data) print("Vocab size: ", opts.vocab_size - 1, " + UNK") print("Words per epoch: ", opts.words_per_epoch) self._examples = examples self._labels = labels self._id2word = opts.vocab_words for i, w in enumerate(self._id2word): self._word2id[w] = i true_logits, sampled_logits = self.forward(examples, labels) loss = self.nce_loss(true_logits, sampled_logits) tf.scalar_summary("NCE loss", loss) self._loss = loss self.optimize(loss) # Properly initialize all variables. tf.initialize_all_variables().run() self.saver = tf.train.Saver() def save_vocab(self): """Save the vocabulary to a file so the model can be reloaded.""" opts = self._options with open(os.path.join(opts.save_path, "vocab.txt"), "w") as f: for i in xrange(opts.vocab_size): f.write("%s %d\n" % (tf.compat.as_text(opts.vocab_words[i]), opts.vocab_counts[i])) def _train_thread_body(self): initial_epoch, = self._session.run([self._epoch]) while True: _, epoch = self._session.run([self._train, self._epoch]) if epoch != initial_epoch: break def train(self): """Train the model.""" opts = self._options initial_epoch, initial_words = self._session.run([self._epoch, self._words]) summary_op = tf.merge_all_summaries() summary_writer = tf.train.SummaryWriter(opts.save_path, graph_def=self._session.graph_def) workers = [] for _ in xrange(opts.concurrent_steps): t = threading.Thread(target=self._train_thread_body) t.start() workers.append(t) last_words, last_time, last_summary_time = initial_words, time.time(), 0 last_checkpoint_time = 0 while True: time.sleep(opts.statistics_interval) # Reports our progress once a while. (epoch, step, loss, words, lr) = self._session.run( [self._epoch, self.global_step, self._loss, self._words, self._lr]) now = time.time() last_words, last_time, rate = words, now, (words - last_words) / ( now - last_time) print("Epoch %4d Step %8d: lr = %5.3f loss = %6.2f words/sec = %8.0f\r" % (epoch, step, lr, loss, rate), end="") sys.stdout.flush() if now - last_summary_time > opts.summary_interval: summary_str = self._session.run(summary_op) summary_writer.add_summary(summary_str, step) last_summary_time = now if now - last_checkpoint_time > opts.checkpoint_interval: self.saver.save(self._session, opts.save_path + "model", global_step=step.astype(int)) last_checkpoint_time = now if epoch != initial_epoch: break for t in workers: t.join() return epoch def _predict(self, analogy): """Predict the top 4 answers for analogy questions.""" idx, = self._session.run([self._analogy_pred_idx], { self._analogy_a: analogy[:, 0], self._analogy_b: analogy[:, 1], self._analogy_c: analogy[:, 2] }) return idx def eval(self): """Evaluate analogy questions and reports accuracy.""" # How many questions we get right at precision@1. correct = 0 total = self._analogy_questions.shape[0] start = 0 while start < total: limit = start + 2500 sub = self._analogy_questions[start:limit, :] idx = self._predict(sub) start = limit for question in xrange(sub.shape[0]): for j in xrange(4): if idx[question, j] == sub[question, 3]: # Bingo! We predicted correctly. E.g., [italy, rome, france, paris]. correct += 1 break elif idx[question, j] in sub[question, :3]: # We need to skip words already in the question. continue else: # The correct label is not the precision@1 break print() print("Eval %4d/%d accuracy = %4.1f%%" % (correct, total, correct * 100.0 / total)) def analogy(self, w0, w1, w2): """Predict word w3 as in w0:w1 vs w2:w3.""" wid = np.array([[self._word2id.get(w, 0) for w in [w0, w1, w2]]]) idx = self._predict(wid) for c in [self._id2word[i] for i in idx[0, :]]: if c not in [w0, w1, w2]: return c return "unknown" def nearby(self, words, num=20): """Prints out nearby words given a list of words.""" ids = np.array([self._word2id.get(x, 0) for x in words]) vals, idx = self._session.run( [self._nearby_val, self._nearby_idx], {self._nearby_word: ids}) for i in xrange(len(words)): print("\n%s\n=====================================" % (words[i])) for (neighbor, distance) in zip(idx[i, :num], vals[i, :num]): print("%-20s %6.4f" % (self._id2word[neighbor], distance)) def _start_shell(local_ns=None): # An interactive shell is useful for debugging/development. import IPython user_ns = {} if local_ns: user_ns.update(local_ns) user_ns.update(globals()) IPython.start_ipython(argv=[], user_ns=user_ns) def main(_): """Train a word2vec model.""" if not FLAGS.train_data or not FLAGS.eval_data or not FLAGS.save_path: print("--train_data --eval_data and --save_path must be specified.") sys.exit(1) opts = Options() with tf.Graph().as_default(), tf.Session() as session: with tf.device("/cpu:0"): model = Word2Vec(opts, session) for _ in xrange(opts.epochs_to_train): model.train() # Process one epoch model.eval() # Eval analogies. # Perform a final save. model.saver.save(session, os.path.join(opts.save_path, "model.ckpt"), global_step=model.global_step) if FLAGS.interactive: # E.g., # [0]: model.analogy('france', 'paris', 'russia') # [1]: model.nearby(['proton', 'elephant', 'maxwell']) _start_shell(locals()) if __name__ == "__main__": tf.app.run()

No description available

# STUDENT'S INFORMATION FROM DATABASE import sqlite3 connection = sqlite3.connect('Credentials.db') cursor = connection.cursor() create_table = "CREATE TABLE IF NOT EXISTS users_credential (id INTEGER PRIMARY KEY," \ "reg_number text," \ "surname text," \ "middle_name text," \ "first_name text," \ "email text,"\ "date_registered text," \ "profile_picture text," \ cursor.execute(create_table) connection.commit() connection.close() #GETTING STUDENTS MARKS AND PREDICTING THEIR CHANCES import numpy as np from models.Prediction.Training import DecisionTreeClassifier returned_grades = list() student_mark = [] model_year = DecisionTreeClassifier() new_input_value = [scores] new_input = np.array(new_input_value) new_input = new_input.reshape(-1, 1) model_year.load_dataset() model_year.encode_variables_for_y(model_year.Y) model_year.spliting_to_training_and_test_set_no_return(model_year.X, model_year.Y) scaled_input = model_year.feature_scaling(new_input) new_prediction = saved_model.predict(scaled_input) if new_prediction == 0: new_prediction = '(0, 5]' elif new_prediction == 1: new_prediction = '(10, 15]' elif new_prediction == 2: new_prediction = '(15, 20]' elif new_prediction == 3: new_prediction = '(20, 25]' elif new_prediction == 4: new_prediction = '(25, 30]' elif new_prediction == 5: new_prediction = '(30, 35]' elif new_prediction == 6: new_prediction = '(35, 40]' elif new_prediction == 7: new_prediction = '(40, 45]' elif new_prediction == 8: new_prediction = '(45, 50]' elif new_prediction == 9: new_prediction = '(5, 10]' elif new_prediction == 10: new_prediction = '(50, 55]' elif new_prediction == 11: new_prediction = '(55, 60]' elif new_prediction == 12: new_prediction = '(60, 65]' elif new_prediction == 13: new_prediction = '(65, 70]' elif new_prediction == 14: new_prediction = '(70, 75]' elif new_prediction == 15: new_prediction = '(75, 80]' elif new_prediction == 16: new_prediction = '(80, 85]' elif new_prediction == 17: new_prediction = '(85, 90]' elif new_prediction == 18: new_prediction = '(90, 95]' elif new_prediction == 19: new_prediction = '(95, 100]' student_mark.append(new_input_value[0]) student_mark.append(new_prediction) returned_grades.append(student_mark) return returned_grades #CLUSTERING import pandas as pd import matplotlib matplotlib.use('Qt4Agg') import numpy as np import matplotlib.pyplot as plt from sklearn.cluster import KMeans from sklearn.preprocessing import StandardScaler dataset = pd.read_csv('dataset/clustering/system_eng_cluster.csv') print(dataset.describe()) print(dataset.get_values()) #K MEAN CLUSTERING import matplotlib import pandas as pd matplotlib.use('Qt4Agg') import matplotlib.pyplot as plt from sklearn.cluster import KMeans class Clustering(object): def __init__(self, csv, ymeans_1=None, ymeans_2=None): self.csv = csv self.ymeans_1 = ymeans_1 self.ymeans_2 = ymeans_2 # importing the dataset with pandas # 'dataset/clustering/system_eng_cluster.csv' self.dataset_loader = pd.read_csv(self.csv) self.X1 = self.dataset_loader.iloc[:, [2, 4]].values self.X2 = self.dataset_loader.iloc[:, [3, 4]].values @staticmethod def process_wcss(x_column_for_wcss): wcss_to_process = [] for i in range(1, 11): kmeans_1 = KMeans(n_clusters=i, init='k-means++', max_iter=300, n_init=10, random_state=0) kmeans_1.fit(x_column_for_wcss) wcss_to_process.append(kmeans_1.inertia_) return wcss_to_process @staticmethod def plot_wcss(wcss_list, course_title): plt.plot(range(1, 11), wcss_list) plt.title("The Elbow Method For Test") plt.xlabel("Number of clusters") plt.ylabel("wcss for {}".format(course_title)) plt.show() plt.imsave() def predict_data(self): # applying k-means to the mall dataset kmeans_predict = KMeans(n_clusters=6, init='k-means++', max_iter=300, n_init=10, random_state=0) self.ymeans_1 = kmeans_predict.fit_predict(self.X1) self.ymeans_2 = kmeans_predict.fit_predict(self.X2) return self.ymeans_1, self.ymeans_2 @staticmethod def visualise_clusters(x_column_to_visualize, y_column_to_visualise, test_title): kmeans_clusters = KMeans(n_clusters=6, init='k-means++', max_iter=300, n_init=10, random_state=0) kmeans_clusters.fit(x_column_to_visualize) # Visualizing the clusters plt.scatter(x_column_to_visualize[y_column_to_visualise == 0, 0], x_column_to_visualize[y_column_to_visualise == 0, 1], s=10, c='red', label='Cluster 1') plt.scatter(x_column_to_visualize[y_column_to_visualise == 1, 0], x_column_to_visualize[y_column_to_visualise == 1, 1], s=10, c='blue', label='Cluster 2') plt.scatter(x_column_to_visualize[y_column_to_visualise == 2, 0], x_column_to_visualize[y_column_to_visualise == 2, 1], s=10, c='green', label='Cluster 3') plt.scatter(x_column_to_visualize[y_column_to_visualise == 3, 0], x_column_to_visualize[y_column_to_visualise == 3, 1], s=10, c='cyan', label='Cluster 4') plt.scatter(x_column_to_visualize[y_column_to_visualise == 4, 0], x_column_to_visualize[y_column_to_visualise == 4, 1], s=10, c='magenta', label='Cluster 5') plt.scatter(x_column_to_visualize[y_column_to_visualise == 5, 0], x_column_to_visualize[y_column_to_visualise == 5, 1], s=10, c='black', label='Cluster 6') plt.scatter(kmeans_clusters.cluster_centers_[:, 0], kmeans_clusters.cluster_centers_[:, 1], s=50, c='yellow', label='Centroids') plt.title("Clusters OF Students Performance Based On Test Score") plt.xlabel("{} SCORE".format(test_title)) plt.ylabel("Test score") plt.legend() plt.show() #QUESTION AND ANSWER SESSIONS TO SEE THE INTEREST OF THE STUDENT import random import pandas as pd from models.aos_questions_and_answer.processedlistofdictionaries import Util # Initializing variables ai_correct = 0 ai_failed = 0 se_correct = 0 se_failed = 0 cn_correct = 0 cn_failed = 0 sye_correct = 0 sye_failed = 0 tc_correct = 0 tc_failed = 0 AI = [] SE = [] CN = [] SYE = [] TC = [] final_scores = [] current_question_number = 0 total_questions = 0 # Reading the CSV file that contains all compiled questions with respective answers dataset = pd.read_csv('models/aos_questions_and_answer/dataset/core_courses.csv') # AI Data processing ai_questions = dataset.iloc[:, :1].values ai_answers = dataset.iloc[:, 1].values ai_list_of_dictionaries_of_questions_and_answers = Util.processed_list_dict(ai_questions, ai_answers) ai_selected_six_random = Util.select_six_random(ai_list_of_dictionaries_of_questions_and_answers) # Software Engineering Data processing software_engineering_questions = dataset.iloc[:, 2:3].values software_engineering_answers = dataset.iloc[:, 3].values software_engineering_list_of_dictionaries_of_questions_and_answers = \ Util.processed_list_dict(software_engineering_questions, software_engineering_answers) se_selected_six_random = Util.select_six_random(software_engineering_list_of_dictionaries_of_questions_and_answers) # Computer Networks Data processing computer_networks_questions = dataset.iloc[:, 4:5].values computer_networks_answers = dataset.iloc[:, 5].values computer_networks_list_of_dictionaries_of_questions_and_answers =\ Util.processed_list_dict(computer_networks_questions, computer_networks_answers) cn_selected_six_random = Util.select_six_random(computer_networks_list_of_dictionaries_of_questions_and_answers) # Systems Engineering Data processing systems_engineering_questions = dataset.iloc[:, 6:7].values systems_engineering_answers = dataset.iloc[:, 7].values systems_engineering_list_of_dictionaries_of_questions_and_answers = \ Util.processed_list_dict(systems_engineering_questions, systems_engineering_answers) sye_selected_six_random = Util.select_six_random(systems_engineering_list_of_dictionaries_of_questions_and_answers) # Theoretical Computing Data processing theoretical_computing_questions = dataset.iloc[:, 8:9].values theoretical_computing_answers = dataset.iloc[:, 9].values theoretical_computing_list_of_dictionaries_of_questions_and_answers = \ Util.processed_list_dict(theoretical_computing_questions, theoretical_computing_answers) tc_selected_six_random = Util.select_six_random(theoretical_computing_list_of_dictionaries_of_questions_and_answers) # Getting total questions and answers to be asked for ever user total_questions_and_answer = Util.all_selected_questions_with_answers(ai_selected_six_random, se_selected_six_random, cn_selected_six_random, sye_selected_six_random, tc_selected_six_random) # print(total_questions_and_answer) for i in total_questions_and_answer.values(): for j in i: total_questions += 1 #APPLICATION FORMS from flask_wtf import FlaskForm from flask_wtf.file import FileField, FileAllowed from flask_login import current_user from models.users.users import User from wtforms import StringField, PasswordField, SubmitField, BooleanField, RadioField, SelectField from wtforms.validators import DataRequired, Length, Email, EqualTo, ValidationError class AdminAddUserForm(FlaskForm): registration_number = StringField('Registration Number', validators=[DataRequired(), Length(min=1, max=20)]) surname = StringField('Surname', validators=[DataRequired(), Length(min=1, max=20)]) middle_name = StringField('Middle Name', validators=[DataRequired(), Length(min=1, max=20)]) first_name = StringField('First Name', validators=[DataRequired(), Length(min=1, max=20)]) email = StringField('Email', validators=[DataRequired(), Email()]) password = PasswordField('Password', validators=[DataRequired(), Length(min=2)]) submit = SubmitField('Register') def validate_email(self, email): _, all_emails_from_database = User.find_all_emails_and_registration_number() if email.data: if email.data in all_emails_from_database: raise ValidationError("That email is taken. Please choose another one!") else: raise ValidationError("This field cannot be blank!") def validate_registration_number(self, registration_number): all_registration_number_from_database, _ = User.find_all_emails_and_registration_number() if registration_number.data: if registration_number.data in all_registration_number_from_database: raise ValidationError("That Registration Number is taken. Please choose another one!") class UserLoginForm(FlaskForm): registration_number = StringField('Registration Number/Username', validators=[DataRequired(), Length(min=1)]) password = PasswordField('Password', validators=[DataRequired(), Length(min=2)]) remember_me = BooleanField('Remember Me') submit = SubmitField('Log In') class UpdateAccountForm(FlaskForm): registration_number = StringField('Registration Number', validators=[DataRequired(), Length(min=1)]) surname = StringField('Surname', validators=[DataRequired(), Length(min=1, max=20)]) middle_name = StringField('Middle Name', validators=[DataRequired(), Length(min=1, max=20)]) first_name = StringField('First Name', validators=[DataRequired(), Length(min=1, max=20)]) password = PasswordField('Password', validators=[DataRequired(), Length(min=2)]) email = StringField('Email', validators=[DataRequired(), Email()]) picture = FileField('Update Profile Picture', validators=[FileAllowed(['jpg', 'png', 'jpeg'])]) submit = SubmitField('Update') def validate_email(self, email): _, all_emails_from_database = User.find_all_emails_and_registration_number() if email.data != current_user.email: if email.data in all_emails_from_database: raise ValidationError("That email is taken. Please choose another one!") class UpdateAdminAccountForm(FlaskForm): registration_number = StringField('Username', validators=[DataRequired(), Length(min=1)]) surname = StringField('Surname', validators=[DataRequired(), Length(min=1, max=20)]) middle_name = StringField('Middle Name', validators=[DataRequired(), Length(min=1, max=20)]) first_name = StringField('First Name', validators=[DataRequired(), Length(min=1, max=20)]) password = PasswordField('Password', validators=[DataRequired(), Length(min=2)]) email = StringField('Email', validators=[DataRequired(), Email()]) picture = FileField('Update Profile Picture', validators=[FileAllowed(['jpg', 'png', 'jpeg'])]) submit = SubmitField('Update') def validate_email(self, email): _, all_emails_from_database = User.find_all_emails_and_registration_number() if email.data != current_user.email: if email.data in all_emails_from_database: raise ValidationError("That email is taken. Please choose another one!") class AdminUpdateStudentAccountForm(FlaskForm): registration_number = StringField('Registration Number', validators=[DataRequired(), Length(min=1)]) surname = StringField('Surname', validators=[DataRequired(), Length(min=1, max=20)]) middle_name = StringField('Middle Name', validators=[DataRequired(), Length(min=1, max=20)]) first_name = StringField('First Name', validators=[DataRequired(), Length(min=1, max=20)]) email = StringField('Email', validators=[DataRequired(), Email()]) submit = SubmitField('Update') def validate_email(self, email): _, all_emails_from_database = User.find_all_emails_and_registration_number() if email.data: if email.data in all_emails_from_database: raise ValidationError("That email is taken. Please choose another one!") class SelectElectiveCourses(FlaskForm): user_type = SelectField('Select Suited Area Of Specialization', validators=[DataRequired()], choices=(("ai", "Artificial Intelligence"), ("cn", "Computer Networks"), ("se", "Software Engineering"), ("sye", "Systems Engineering"))) submit = SubmitField('START TEST') class StartQuiz(FlaskForm): submit = SubmitField('START TEST') class QuestionForm(FlaskForm): question_option = RadioField("Answers", coerce=str) submit_next = SubmitField('NEXT') # submit_previous = SubmitField('PREVIOUS') #USER LOGIN SYSTEM import datetime import sqlite3 import uuid from flask_login import UserMixin from extensions import login_manager from utils import Utils @login_manager.user_loader def load_user(user_id): return User.find_by_id(user_id) class User(UserMixin): date_time = str(datetime.datetime.utcnow()).split() date, time = date_time date = str(date) time = time.split(".") time = time[0].__str__() def __init__(self, inc_id=None, reg_number=None, surname=None, middle_name=None, first_name=None, email=None, password=None, _id=None, timestamp=time, date=date, default_image=None, account_type=None): self.inc_id = inc_id self.reg_number = reg_number self.surname = surname self.middle_name = middle_name self.first_name = first_name self.email = email self.password = password self.id = uuid.uuid4().__str__() if _id is None else _id self.timestamp = timestamp self.date_registered = date self.default_image = "default.png" if default_image is None else default_image self.account_type = account_type def save_to_db(self): """ This saves the question to the database Returns: A notification string """ connection = sqlite3.connect("./database/Credentials.db") cursor = connection.cursor() query = "INSERT INTO users_credential VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" cursor.execute(query, (self.reg_number, self.surname, self.middle_name, self.first_name, self.email, self.password, self.id, self.timestamp, self.date_registered, self.default_image, self.account_type,)) connection.commit() connection.close() def create_admin(self, surname, middle_name, first_name, email, password, username): self.account_type = "admin" connection = sqlite3.connect("./database/credentials.db") cursor = connection.cursor() query = "INSERT INTO users_credential VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" cursor.execute(query, (username, surname, middle_name, first_name, email, password, self.id, self.timestamp, self.date_registered, self.default_image, self.account_type,)) connection.commit() connection.close() def insert_student_into_db(self, surname, middle_name, first_name, reg_number, email, password): encrypted_password = Utils.encrypt_password(password=password) self.account_type = "student" connection = sqlite3.connect("./database/credentials.db") cursor = connection.cursor() query = "INSERT INTO users_credential VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" cursor.execute(query, (reg_number, surname, middle_name, first_name, email, encrypted_password, self.id, self.timestamp, self.date_registered, self.default_image, self.account_type,)) connection.commit() connection.close() @staticmethod def update_profile(username, surname, middle_name, first_name, password, email, picture_to_update, user_corresponding_id): encrypted_password = Utils.encrypt_password(password=password) connection = sqlite3.connect('./database/Credentials.db') cursor = connection.cursor() query = "UPDATE users_credential SET reg_number=?, surname=?, middle_name=?, first_name=?, password=?," \ "email=?, profile_picture=? WHERE _id=?" cursor.execute(query, (username,surname, middle_name, first_name, encrypted_password, email, picture_to_update, user_corresponding_id,)) connection.commit() connection.close() @staticmethod def update_student_profile_by_admin(reg_number, surname, middle_name, first_name, email, user_corresponding_id): connection = sqlite3.connect('./database/Credentials.db') cursor = connection.cursor() query = "UPDATE users_credential SET reg_number=?, surname=?, middle_name=?, first_name=?," \ "email=? WHERE _id=?" cursor.execute(query, (reg_number, surname, middle_name, first_name, email, user_corresponding_id,)) connection.commit() connection.close() @staticmethod def update_password(new_password, user_corresponding_id): connection = sqlite3.connect('./database/Credentials.db') cursor = connection.cursor() query = "UPDATE users_credential SET password=? WHERE _id=?" cursor.execute(query, (new_password, user_corresponding_id,)) connection.commit() connection.close() @staticmethod def update_email(email_update, user_corresponding_id): connection = sqlite3.connect('./database/Credentials.db') cursor = connection.cursor() query = "UPDATE users_credential SET email=? WHERE _id=?" cursor.execute(query, (email_update, user_corresponding_id,)) connection.commit() connection.close() @staticmethod def update_profile_picture(picture_file, user_corresponding_id): connection = sqlite3.connect('./database/Credentials.db') cursor = connection.cursor() query = "UPDATE users_credential SET profile_picture=? WHERE _id=?" cursor.execute(query, (picture_file, user_corresponding_id,)) connection.commit() connection.close() @classmethod def find_by_registration_number(cls, reg_number): connection = sqlite3.connect('./database/Credentials.db') cursor = connection.cursor() query = "SELECT * FROM users_credential WHERE reg_number=?" result = cursor.execute(query, (reg_number,)) row = result.fetchone() if row: user = cls(*row) # same as row[0], row[1], row[2]...passing args by position else: user = None connection.close() return user @classmethod def find_by_email(cls, email): connection = sqlite3.connect('./database/Credentials.db') cursor = connection.cursor() query = "SELECT * FROM users_credential WHERE email=?" result = cursor.execute(query, (email,)) row = result.fetchone() if row: user = cls(*row) # same as row[0], row[1], row[2]...passing args by position else: user = None connection.close() return user @staticmethod def find_all_emails_and_registration_number(): connection = sqlite3.connect('./database/Credentials.db') cursor = connection.cursor() query = "SELECT * FROM users_credential ORDER BY email ASC " result = cursor.execute(query, ) rows = result.fetchall() new_registration_number = [] new_email = [] for row in rows: new_registration_number.append(row[1]) new_email.append(row[5]) return new_registration_number, new_email @classmethod def find_by_id(cls, _id): connection = sqlite3.connect('./database/Credentials.db') cursor = connection.cursor() query = "SELECT * FROM users_credential WHERE _id=?" result = cursor.execute(query, (_id,)) row = result.fetchone() if row: user = cls(*row) # same as row[0], row[1], row[2]...passing args by position else: user = None connection.close() return user @classmethod def fetch_all_students_by_account_type(cls): student = [] connection = sqlite3.connect('./database/Credentials.db') cursor = connection.cursor() query = "SELECT * FROM users_credential WHERE account_type='student'" result = cursor.execute(query,) rows = result.fetchall() if rows: for row in rows: student.append(row) else: student = [] connection.close() return student #APP CREATION AND RATING BY STUDENTS from flask import Flask # Blueprints Imports from blueprints.page import page from blueprints.users import user from blueprints.questions import aos_test, elective_course # extensions Import from extensions import mail, csrf, login_manager CELERY_TASK_LIST = ['blueprints.contact.tasks', ] # app = Flask(__name__, instance_relative_config=True) # app.config.from_object('config.settings') # app.config.from_pyfile('settings.py', silent=True) # # app.register_blueprint(page) def create_app(settings_override=None): """ Create a Flask application using the app factory pattern. :param settings_override: Override settings :return: Flask app """ application = Flask(__name__, instance_relative_config=True) application.config.from_object('config.settings') application.config.from_pyfile('settings.py', silent=True) if settings_override: application.config.update(settings_override) application.register_blueprint(page) application.register_blueprint(user) application.register_blueprint(aos_test) application.register_blueprint(elective_course) extensions(application) return application def extensions(our_app): mail.init_app(our_app) csrf.init_app(our_app) login_manager.init_app(our_app) login_manager.login_view = 'user.login' login_manager.login_message_category = 'info' return None #CONTACT US from flask_mail import Mail from flask_wtf import CSRFProtect from flask_login import LoginManager mail = Mail() csrf = CSRFProtect() login_manager = LoginManager() #HOW TO USE from app import create_app from models.users.users import User from utils import Utils if __name__ == '__main__': app = create_app() with open("first_time_server_run.txt", "r") as new_file: content = new_file.read() if content == "": var = True while var: print("Welcome Admin Please put in the following Credentials") surname = input("Surname: ") middle_name = input("Middle Name: ") first_name = input("First Name: ") user_name = input("Username: ") email = input("E-mail: ") password = input("Password: ") if surname != "" and middle_name != "" and first_name != "" and email != "" and user_name != "" \ and password != "": encrypted_password = Utils.encrypt_password(password) grand_admin = User() grand_admin.create_admin(surname=surname, middle_name=middle_name, email=email, first_name=first_name, password=encrypted_password, username=user_name) with open("first_time_server_run.txt", "a") as new_file_write: new_file_write.write("true") var = False break else: continue app.run() #UTILS from passlib.hash import pbkdf2_sha512 import constants import re class Utils(object): @staticmethod def encrypt_password(password): return pbkdf2_sha512.encrypt(password) @staticmethod def check_encrypted_password(password, hashed_password): return pbkdf2_sha512.verify(password, hashed_password) @staticmethod def allowed_file(filename): return '.' in filename and \ filename.rsplit('.', 1)[1].lower() in constants.ALLOWED_EXTENSIONS @staticmethod def strong_password(password_to_check): a = b = c = d = e = f = '' try: matcher_digits = re.compile(r'[0-9]+') matcher_lowercase = re.compile(r'[a-z]+') matcher_uppercase = re.compile(r'[A-Z]+') matcher_special = re.compile(r'[\W.\\?\[\]|+*$()_^{\}]+') mo_digits = matcher_digits.search(password_to_check) mo_lowercase = matcher_lowercase.search(password_to_check) mo_uppercase = matcher_uppercase.search(password_to_check) mo_special = matcher_special.search(password_to_check) if mo_digits and mo_lowercase and mo_uppercase and mo_special: return None if not mo_digits or not mo_lowercase or not mo_uppercase or not mo_special: if not mo_special: a += "one special character is required" if not mo_digits: b += "a number is required" if not mo_lowercase: c += "a lowercase letter is required" if not mo_uppercase: d += "an uppercase letter is required" if not mo_digits and not mo_lowercase and not mo_uppercase and not mo_special: e += "Password should include a Lowercase, a Uppercase, Numbers and special characters" return a, b, c, d, e except Exception as _: f += "Password should include a Lowercase, a Uppercase, Numbers and special characters" return f @staticmethod def check_reg_number(reg_num): try: matcher = re.compile(r'\d{4}/\d{6}') matching_reg_number = matcher.search(reg_num) reg_num_format_length = reg_num.split("/") reg_num_format_length_first = reg_num_format_length[0] reg_num_format_length_last = reg_num_format_length[1] if matching_reg_number and \ len(reg_num_format_length_first) == 4 and \ len(reg_num_format_length_last) == 6 and \ len(reg_num_format_length) == 2: return None else: return "Incorrect formatted Registration Number" except Exception as _: return "Incorrect formatted Registration Number"